Skip to content
Snippets Groups Projects
Commit ce70016d authored by Mingchung Xia's avatar Mingchung Xia
Browse files

Code cleanup and documentation

parent a23a1d1c
No related branches found
No related tags found
1 merge request!13HNSW Implementation with Testcases
Hide whitespace changes
Inline Side-by-side
Showing
with 264 additions and 250 deletions
Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Codable.swift
+ 41
− 41
View file @ ce70016d
// Copyright (c) 2024 Jim Wallace
//// Copyright (c) 2024 Jim Wallace
////
//// Permission is hereby granted, free of charge, to any person
//// obtaining a copy of this software and associated documentation
//// files (the "Software"), to deal in the Software without
//// restriction, including without limitation the rights to use,
//// copy, modify, merge, publish, distribute, sublicense, and/or sell
//// copies of the Software, and to permit persons to whom the
//// Software is furnished to do so, subject to the following
//// conditions:
////
//// The above copyright notice and this permission notice shall be
//// included in all copies or substantial portions of the Software.
////
//// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
//// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
//// OTHER DEALINGS IN THE SOFTWARE.
////
//// Created by Mingchung Xia on 2024-02-07.
////
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//import Foundation
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//// MARK: Decodable conformance is in HNSWCorpus
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
// Created by Mingchung Xia on 2024-02-07.
//
import Foundation
// MARK: Decodable conformance is in HNSWCorpus
extension EphemeralHNSWCorpus: Codable {
enum CodingKeys: String, CodingKey {
case _documentEncoder
case encodedDocuments
case dictionary
}
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(_documentEncoder, forKey: ._documentEncoder)
try container.encode(encodedDocuments, forKey: .encodedDocuments)
try container.encode(dictionary, forKey: .dictionary)
}
}
//extension EphemeralHNSWCorpus: Codable {
// enum CodingKeys: String, CodingKey {
// case _documentEncoder
// case encodedDocuments
// case dictionary
// }
//
// func encode(to encoder: Encoder) throws {
// var container = encoder.container(keyedBy: CodingKeys.self)
// try container.encode(_documentEncoder, forKey: ._documentEncoder)
// try container.encode(encodedDocuments, forKey: .encodedDocuments)
// try container.encode(dictionary, forKey: .dictionary)
// }
//}
Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus.swift
+ 12
− 7
View file @ ce70016d
......@@ -21,6 +21,11 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
// The HNSW work is based on the original work of Jaden Geller
// See the https://github.com/JadenGeller/similarity-topology.git
// for reference. The code is used with permission from the author
// under the MIT License.
//
// Created by Mingchung Xia on 2024-02-14.
//
......@@ -54,13 +59,13 @@ final class EphemeralHNSWCorpus<Scalar: BinaryFloatingPoint & Codable>: SNLPCorp
encodedDocuments = DeterministicEphemeralVectorIndex<[Scalar]>(typicalNeighborhoodSize: typicalNeighborhoodSize)
}
// Decodable conformance
required init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
_documentEncoder = try container.decode(ContextFreeEncoder<Scalar>.self, forKey: ._documentEncoder)
encodedDocuments = try container.decode(DeterministicEphemeralVectorIndex<[Scalar]>.self, forKey: .encodedDocuments)
dictionary = try container.decode(HNSWDictionary.self, forKey: .dictionary)
}
// // Decodable conformance
// required init(from decoder: Decoder) throws {
// let container = try decoder.container(keyedBy: CodingKeys.self)
// _documentEncoder = try container.decode(ContextFreeEncoder<Scalar>.self, forKey: ._documentEncoder)
// encodedDocuments = try container.decode(DeterministicEphemeralVectorIndex<[Scalar]>.self, forKey: .encodedDocuments)
// dictionary = try container.decode(HNSWDictionary.self, forKey: .dictionary)
// }
@inlinable
func addUntokenizedDocument(_ document: String) {
......
Sources/SwiftNLP/1. Data Collection/HNSW/DeterministicEphemeralVectorIndex + Codable.swift
+ 50
− 50
View file @ ce70016d
// Copyright (c) 2024 Jim Wallace
//// Copyright (c) 2024 Jim Wallace
////
//// Permission is hereby granted, free of charge, to any person
//// obtaining a copy of this software and associated documentation
//// files (the "Software"), to deal in the Software without
//// restriction, including without limitation the rights to use,
//// copy, modify, merge, publish, distribute, sublicense, and/or sell
//// copies of the Software, and to permit persons to whom the
//// Software is furnished to do so, subject to the following
//// conditions:
////
//// The above copyright notice and this permission notice shall be
//// included in all copies or substantial portions of the Software.
////
//// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
//// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
//// OTHER DEALINGS IN THE SOFTWARE.
////
//// Created by Mingchung Xia on 2024-02-07.
////
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//import Foundation
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//extension DeterministicEphemeralVectorIndex: Encodable where Vector: Encodable {
// enum CodingKeys: String, CodingKey {
// case typicalNeighborhoodSize
// case vectors
// }
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
// public func encode(to encoder: Encoder) throws {
// var container = encoder.container(keyedBy: CodingKeys.self)
// try container.encode(typicalNeighborhoodSize, forKey: .typicalNeighborhoodSize)
// try container.encode(base.vectors, forKey: .vectors)
// }
//}
//
//extension DeterministicEphemeralVectorIndex: Decodable where Vector: Decodable {
// public init(from decoder: Decoder) throws {
// let container = try decoder.container(keyedBy: CodingKeys.self)
// let typicalNeighborhoodSize = try container.decode(Int.self, forKey: .typicalNeighborhoodSize)
// let vectors = try container.decode([Vector].self, forKey: .vectors)
//
// self.init(typicalNeighborhoodSize: typicalNeighborhoodSize)
// for vector in vectors {
// self.insert(vector)
// }
// }
//}
//
// Created by Mingchung Xia on 2024-02-07.
//
import Foundation
extension DeterministicEphemeralVectorIndex: Encodable where Vector: Encodable {
enum CodingKeys: String, CodingKey {
case typicalNeighborhoodSize
case vectors
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(typicalNeighborhoodSize, forKey: .typicalNeighborhoodSize)
try container.encode(base.vectors, forKey: .vectors)
}
}
extension DeterministicEphemeralVectorIndex: Decodable where Vector: Decodable {
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
let typicalNeighborhoodSize = try container.decode(Int.self, forKey: .typicalNeighborhoodSize)
let vectors = try container.decode([Vector].self, forKey: .vectors)
self.init(typicalNeighborhoodSize: typicalNeighborhoodSize)
for vector in vectors {
self.insert(vector)
}
}
}
Sources/SwiftNLP/1. Data Collection/HNSW/HNSWCorpusDataHandler.swift
+ 149
− 149
View file @ ce70016d
// Copyright (c) 2024 Jim Wallace
//// Copyright (c) 2024 Jim Wallace
////
//// Permission is hereby granted, free of charge, to any person
//// obtaining a copy of this software and associated documentation
//// files (the "Software"), to deal in the Software without
//// restriction, including without limitation the rights to use,
//// copy, modify, merge, publish, distribute, sublicense, and/or sell
//// copies of the Software, and to permit persons to whom the
//// Software is furnished to do so, subject to the following
//// conditions:
////
//// The above copyright notice and this permission notice shall be
//// included in all copies or substantial portions of the Software.
////
//// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
//// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
//// OTHER DEALINGS IN THE SOFTWARE.
////
//// Created by Mingchung Xia on 2024-02-13.
////
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//// MARK: This is outdated since we now have the presence of a DurableHNSWCorpus but still available for reference
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//import Foundation
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//final class HNSWCorpusDataHandler<Scalar: BinaryFloatingPoint & Codable> {
// var corpus: HNSWCorpus<Scalar>
// private var url: URL?
//
// init(corpus: HNSWCorpus<Scalar>, resource: String = "hnsw") {
// self.corpus = corpus
//// self.url = Bundle.module.url(forResource: resource, withExtension: "mmap")
// if let downloadsDirectory = FileManager.default.urls(for: .downloadsDirectory, in: .userDomainMask).first {
// self.url = downloadsDirectory.appendingPathComponent(resource + ".mmap")
// }
// }
//
// /// It is very difficult to get the exact size of the corpus as every class also depends on other classes
// /// The size of the memory map may not even be correct if it only stores the vectors, and the vectors are really the only "important" part
// func getCorpusSize() -> Int {
//// return heapSize(corpus)
//// return class_getInstanceSize(type(of: corpus))
//// return MemoryLayout.size(ofValue: corpus)
// var size = 0
// let data = corpus.encodedDocuments.base.vectors
// for vector in data {
// size += MemoryLayout.size(ofValue: vector)
// }
// return size
// }
//
// func getDictionarySize(includeKey: Bool = true) -> Int {
// var size = 0
// let data = corpus.getDictionary()
// for (key, documentVectorPair) in data {
// if includeKey { size += MemoryLayout.size(ofValue: key) }
// size += MemoryLayout.size(ofValue: documentVectorPair.untokenizedDocument)
// size += MemoryLayout.size(ofValue: documentVectorPair.vector)
// }
// return size
// }
//
// private func heapSize(_ obj: AnyObject) -> Int {
// return malloc_size(Unmanaged.passUnretained(obj).toOpaque())
// }
//}
//
// Created by Mingchung Xia on 2024-02-13.
//
// MARK: This is outdated since we now have the presence of a DurableHNSWCorpus but still available for reference
import Foundation
final class HNSWCorpusDataHandler<Scalar: BinaryFloatingPoint & Codable> {
var corpus: HNSWCorpus<Scalar>
private var url: URL?
init(corpus: HNSWCorpus<Scalar>, resource: String = "hnsw") {
self.corpus = corpus
// self.url = Bundle.module.url(forResource: resource, withExtension: "mmap")
if let downloadsDirectory = FileManager.default.urls(for: .downloadsDirectory, in: .userDomainMask).first {
self.url = downloadsDirectory.appendingPathComponent(resource + ".mmap")
}
}
/// It is very difficult to get the exact size of the corpus as every class also depends on other classes
/// The size of the memory map may not even be correct if it only stores the vectors, and the vectors are really the only "important" part
func getCorpusSize() -> Int {
// return heapSize(corpus)
// return class_getInstanceSize(type(of: corpus))
// return MemoryLayout.size(ofValue: corpus)
var size = 0
let data = corpus.encodedDocuments.base.vectors
for vector in data {
size += MemoryLayout.size(ofValue: vector)
}
return size
}
func getDictionarySize(includeKey: Bool = true) -> Int {
var size = 0
let data = corpus.getDictionary()
for (key, documentVectorPair) in data {
if includeKey { size += MemoryLayout.size(ofValue: key) }
size += MemoryLayout.size(ofValue: documentVectorPair.untokenizedDocument)
size += MemoryLayout.size(ofValue: documentVectorPair.vector)
}
return size
}
private func heapSize(_ obj: AnyObject) -> Int {
return malloc_size(Unmanaged.passUnretained(obj).toOpaque())
}
}
extension HNSWCorpusDataHandler {
func saveMemoryMap() {
guard let url = url else {
print("URL to resource not found")
return
}
let fileManager = FileManager.default
if !fileManager.fileExists(atPath: url.path) {
fileManager.createFile(atPath: url.path, contents: nil, attributes: nil)
}
do {
// let fileHandle = try FileHandle(forWritingTo: url)
//extension HNSWCorpusDataHandler {
// func saveMemoryMap() {
// guard let url = url else {
// print("URL to resource not found")
// return
// }
// let fileManager = FileManager.default
// if !fileManager.fileExists(atPath: url.path) {
// fileManager.createFile(atPath: url.path, contents: nil, attributes: nil)
// }
// do {
//// let fileHandle = try FileHandle(forWritingTo: url)
////
//// let count = corpus.count
//// let countData = withUnsafeBytes(of: count) { Data($0) }
//// fileHandle.write(countData)
////
//// for pair in corpus {
//// let documentData = pair.untokenizedDocument.utf8CString.withUnsafeBufferPointer { Data(buffer: $0) }
//// fileHandle.write(documentData)
//// }
//// fileHandle.closeFile()
//
// let count = corpus.count
// let countData = withUnsafeBytes(of: count) { Data($0) }
// fileHandle.write(countData)
//
// for pair in corpus {
// let documentData = pair.untokenizedDocument.utf8CString.withUnsafeBufferPointer { Data(buffer: $0) }
// fileHandle.write(documentData)
// }
// fileHandle.closeFile()
print("Saving HNSW to file...")
/// Using the Codable conformances
let encoder = JSONEncoder()
let encoded = try encoder.encode(corpus)
try encoded.write(to: url)
} catch {
print("Error writing HNSW to file: \(error)")
}
}
/// This saves only the untokenized documents dictionary map
func saveDictionaryMemoryMap() {
// TODO: Move from DurableHNSW extension once HNSW wrapper is created
}
// TODO: find out how to not rebuild the index
static func loadMemoryMap(encoder: any SNLPEncoder, typicalNeighborhoodSize: Int = 20, resource: String = "hnsw") -> HNSWCorpus<Double> {
guard let url = Bundle.module.url(forResource: resource, withExtension: "mmap") else {
print("URL to resource not found")
return HNSWCorpus(encoder: encoder, typicalNeighborhoodSize: typicalNeighborhoodSize)
}
var loadedCorpus = HNSWCorpus(encoder: encoder, typicalNeighborhoodSize: typicalNeighborhoodSize)
do {
// let data = try Data(contentsOf: url, options: .alwaysMapped)
// let countData = data.prefix(MemoryLayout<Int>.size)
// let count: Int = countData.withUnsafeBytes { $0.load(as: Int.self) }
// var index = MemoryLayout<Int>.size
//
// for _ in 0..<count {
// if let stringRange = data[index...].range(of: "\0".data(using: .utf8)!) {
// let documentData = data[index..<stringRange.lowerBound]
// if let document = String(data: documentData, encoding: .utf8) {
// // Add the untokenized document to the corpus
// loadedCorpus.addUntokenizedDocument(document)
// index = stringRange.upperBound
// }
// } else {
// break
// }
// }
/// Using the Codable conformances
print("Loading HNSW from file...")
let decoder = JSONDecoder()
let data = try Data(contentsOf: url)
loadedCorpus = try decoder.decode(HNSWCorpus<Double>.self, from: data)
} catch {
print("Error reading HNSW from file: \(error)")
}
return loadedCorpus
}
static func loadMemoryMap(encoding: ContextFreeEncoder<Scalar>.PreComputedEmbeddings, typicalNeighborhoodSize: Int = 20, resource: String = "hnsw") -> HNSWCorpus<Double> {
let encoder = ContextFreeEncoder<Scalar>(source: encoding)
return loadMemoryMap(encoder: encoder, typicalNeighborhoodSize: typicalNeighborhoodSize, resource: resource)
}
}
// print("Saving HNSW to file...")
// /// Using the Codable conformances
// let encoder = JSONEncoder()
// let encoded = try encoder.encode(corpus)
// try encoded.write(to: url)
// } catch {
// print("Error writing HNSW to file: \(error)")
// }
// }
//
// /// This saves only the untokenized documents dictionary map
// func saveDictionaryMemoryMap() {
// // TODO: Move from DurableHNSW extension once HNSW wrapper is created
// }
//
// // TODO: find out how to not rebuild the index
// static func loadMemoryMap(encoder: any SNLPEncoder, typicalNeighborhoodSize: Int = 20, resource: String = "hnsw") -> HNSWCorpus<Double> {
// guard let url = Bundle.module.url(forResource: resource, withExtension: "mmap") else {
// print("URL to resource not found")
// return HNSWCorpus(encoder: encoder, typicalNeighborhoodSize: typicalNeighborhoodSize)
// }
//
// var loadedCorpus = HNSWCorpus(encoder: encoder, typicalNeighborhoodSize: typicalNeighborhoodSize)
//
// do {
//// let data = try Data(contentsOf: url, options: .alwaysMapped)
//// let countData = data.prefix(MemoryLayout<Int>.size)
//// let count: Int = countData.withUnsafeBytes { $0.load(as: Int.self) }
//// var index = MemoryLayout<Int>.size
////
//// for _ in 0..<count {
//// if let stringRange = data[index...].range(of: "\0".data(using: .utf8)!) {
//// let documentData = data[index..<stringRange.lowerBound]
//// if let document = String(data: documentData, encoding: .utf8) {
//// // Add the untokenized document to the corpus
//// loadedCorpus.addUntokenizedDocument(document)
//// index = stringRange.upperBound
//// }
//// } else {
//// break
//// }
//// }
//
// /// Using the Codable conformances
// print("Loading HNSW from file...")
// let decoder = JSONDecoder()
// let data = try Data(contentsOf: url)
// loadedCorpus = try decoder.decode(HNSWCorpus<Double>.self, from: data)
// } catch {
// print("Error reading HNSW from file: \(error)")
// }
// return loadedCorpus
// }
//
// static func loadMemoryMap(encoding: ContextFreeEncoder<Scalar>.PreComputedEmbeddings, typicalNeighborhoodSize: Int = 20, resource: String = "hnsw") -> HNSWCorpus<Double> {
// let encoder = ContextFreeEncoder<Scalar>(source: encoding)
// return loadMemoryMap(encoder: encoder, typicalNeighborhoodSize: typicalNeighborhoodSize, resource: resource)
// }
//}
Sources/SwiftNLP/1. Data Collection/HNSW/RNG/MersenneTwisterRNG.swift
+ 4
− 0
View file @ ce70016d
......@@ -21,6 +21,10 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
// See the https://github.com/JadenGeller/similarity-topology.git
// for reference. The code is used with permission from the author
// under the MIT License.
//
// Created by Mingchung Xia on 2024-01-28.
//
......
Tests/SwiftNLPTests/1. Data Collection/HNSW/DurableHNSWCorpusTests.swift
+ 8
− 0
View file @ ce70016d
......@@ -34,6 +34,14 @@ final class DurableHNSWCorpusTests: XCTestCase {
let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
workingDirectoryPath = FilePath(directoryURL.path)
/// This commented out code alternatively works in the XCode bundle resource environment
// guard let resourcesPath = Bundle.module.resourcePath else { fatalError("Failed to find resource path.") }
// let resourcesDirectoryURL = URL(fileURLWithPath: resourcesPath).appendingPathComponent("lmdb")
// let fileManager = FileManager.default
// try fileManager.createDirectory(at: resourcesDirectoryURL, withIntermediateDirectories: true, attributes: nil)
// print("Resources directory: \(resourcesDirectoryURL)")
// workingDirectoryPath = FilePath(resourcesDirectoryURL.path)
}
func testBasicExample() throws {
......
Tests/SwiftNLPTests/1. Data Collection/HNSW/EphemeralHNSWCorpusTests.swift
+ 0
− 3
View file @ ce70016d
......@@ -5,9 +5,6 @@ import System
@testable import SwiftNLP
final class EphemeralHNSWCorpusTests: XCTestCase {
// MARK: There is also an HNSWCorpusDataHandler class which can store an EphemeralHNSWCorpus into a memory map
/// However, it is not recommended to use this for large datasets since it uses a currently slow coding protocol conformance with JSONEncoder/Decoder
// MARK: EphemeralHNSWCorpus can also be used as its typealias HNSWCorpus
// Load a small set of documents and confirm that corpus and dictionary are updated accordingly
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment