diff --git a/Sources/SwiftNLP/1. Data Collection/DurableHNSWCorpus.swift b/Sources/SwiftNLP/1. Data Collection/DurableHNSWCorpus.swift
index e7a117ee5d663d869e9b5757da2374c85e58ad86..ad1ce997f056d5cb6ea794b0c673d7ec41c0788c 100644
--- a/Sources/SwiftNLP/1. Data Collection/DurableHNSWCorpus.swift
+++ b/Sources/SwiftNLP/1. Data Collection/DurableHNSWCorpus.swift
@@ -9,7 +9,7 @@ import Foundation
import CoreLMDB
import CoreLMDBCoders
-class DurableHNSWCorpus/*<Scalar: BinaryFloatingPoint & Codable>: SNLPCorpus*/ {
+final class DurableHNSWCorpus/*<Scalar: BinaryFloatingPoint & Codable>: SNLPCorpus*/ {
public typealias Scalar = Double /// This is a placeholder to make things work easier right now
public typealias HNSWDictionary = [Int: DocumentVectorPair]
diff --git a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Codable.swift b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Codable.swift
similarity index 93%
rename from Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Codable.swift
rename to Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Codable.swift
index aee1feba2057eee9b0e7ae66614c7aa0984a69c0..b5bc0eea3206d4b4c80d89bcd5c36244bf661c21 100644
--- a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Codable.swift
+++ b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Codable.swift
@@ -9,7 +9,7 @@ import Foundation
// MARK: Decodable conformance is in HNSWCorpus
-extension HNSWCorpus: Codable {
+extension EphemeralHNSWCorpus: Codable {
enum CodingKeys: String, CodingKey {
case _documentEncoder
case encodedDocuments
diff --git a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Dictionary.swift b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Dictionary.swift
similarity index 94%
rename from Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Dictionary.swift
rename to Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Dictionary.swift
index e8fb97347ba1cb821b34b2bade98132711186e9e..489b4467d765fdbe46522b44d19bbdef25928d1d 100644
--- a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Dictionary.swift
+++ b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Dictionary.swift
@@ -7,7 +7,7 @@
import Foundation
-extension HNSWCorpus {
+extension EphemeralHNSWCorpus {
/// This extension is used for the dictionary operations
public struct DocumentVectorPair {
var untokenizedDocument: String
@@ -50,7 +50,7 @@ extension HNSWCorpus {
}
}
-extension HNSWCorpus.DocumentVectorPair: Codable where Scalar: Codable {
+extension EphemeralHNSWCorpus.DocumentVectorPair: Codable where Scalar: Codable {
enum CodingKeys: String, CodingKey {
case untokenizedDocument
case vector
diff --git a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Sequence.swift b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Sequence.swift
similarity index 93%
rename from Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Sequence.swift
rename to Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Sequence.swift
index 554ed70d3e67af7883ae4111b4fa3ab7e357dc71..515db3b045fddc38859394e083d72fe7d9b3be78 100644
--- a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus + Sequence.swift
+++ b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus + Sequence.swift
@@ -21,9 +21,9 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
-/// It may be more useful to make the conformances based on the dictionary instead of encodedDocuments
+/// HNSWCorpus iterates through its dictionary of key to document vector pairs
-extension HNSWCorpus: Sequence, Collection {
+extension EphemeralHNSWCorpus: Sequence, Collection {
typealias Element = [Scalar]
diff --git a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus.swift b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus.swift
similarity index 94%
rename from Sources/SwiftNLP/1. Data Collection/HNSWCorpus.swift
rename to Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus.swift
index c75afd120ea3e535e6d71943d020f67ba49fbbdd..dbcde20d67c1d5e2e083b136064b76f59af18186 100644
--- a/Sources/SwiftNLP/1. Data Collection/HNSWCorpus.swift
+++ b/Sources/SwiftNLP/1. Data Collection/EphemeralHNSWCorpus.swift
@@ -23,7 +23,10 @@
import Foundation
-class HNSWCorpus<Scalar: BinaryFloatingPoint & Codable>: SNLPCorpus {
+// MARK: Allow EphemeralHNSWCorpus to simply be used as HNSWCorpus
+typealias HNSWCorpus = EphemeralHNSWCorpus
+
+final class EphemeralHNSWCorpus<Scalar: BinaryFloatingPoint & Codable>: SNLPCorpus {
public typealias HNSWDictionary = [Int: DocumentVectorPair]
diff --git a/Sources/SwiftNLP/1. Data Collection/HNSW/DeterministicDurableVectorIndex.swift b/Sources/SwiftNLP/1. Data Collection/HNSW/DeterministicDurableVectorIndex.swift
index 5ea8677c02b0e174710ae2c09ff57e72444aadcc..cc367bb6fb6c429fe1eccea7ca078bf317c41e09 100644
--- a/Sources/SwiftNLP/1. Data Collection/HNSW/DeterministicDurableVectorIndex.swift
+++ b/Sources/SwiftNLP/1. Data Collection/HNSW/DeterministicDurableVectorIndex.swift
@@ -42,17 +42,15 @@ extension DurableVectorIndex {
public struct DeterministicDurableVectorIndex/*<Vector: Collection & Codable> where Vector.Element: BinaryFloatingPoint*/ {
public typealias Vector = [Double]
public typealias Index = DurableVectorIndex<CartesianDistanceMetric<Vector>, Vector.Element>
-// public typealias Index = DurableVectorIndex<CosineSimilarityMetric<Vector>, Vector.Element>
public var base: Index
public var typicalNeighborhoodSize: Int
- public var size: Int = 0
+ public var size: Int = 0 // TODO: This size is not set when read from LMDB
private var srng = SeedableRandomNumberGenerator(seed: 1)
// private var drng = DeterministicRandomNumberGenerator(seed: 1)
public init(namespace: String, typicalNeighborhoodSize: Int = 20, in transaction: Transaction) throws {
let metric = CartesianDistanceMetric<Vector>()
-// let metric = CosineSimilarityMetric<Vector>()
let config = Config.unstableDefault(typicalNeighborhoodSize: typicalNeighborhoodSize)
self.base = try Index(
namespace: namespace,
diff --git a/Tests/SwiftNLPTests/2. Encoding/DurableHNSWCorpusTests.swift b/Tests/SwiftNLPTests/2. Encoding/DurableHNSWCorpusTests.swift
new file mode 100644
index 0000000000000000000000000000000000000000..4360f27a0f1ce31d9cdc4228dd0902d1c59c9e83
--- /dev/null
+++ b/Tests/SwiftNLPTests/2. Encoding/DurableHNSWCorpusTests.swift
@@ -0,0 +1,217 @@
+#if os(macOS)
+import XCTest
+import Foundation
+import CoreLMDB
+import System
+@testable import SwiftNLP
+
+final class DurableHNSWCorpusTests: XCTestCase {
+
+ /// Setting up constants for environment
+ private let ONE_GB: Int = 1_073_741_824
+ private let ONE_MB: Int = 1_048_576
+ private let ONE_KB: Int = 1_024
+ private let ONE_B: Int = 1
+ private let DEFAULT_MAXREADERS: UInt32 = 126
+ private let DEFAULT_MAXDBS: UInt32 = 10
+
+ /// Setting up working directory
+ private var workingDirectoryPath: FilePath!
+
+ override func setUpWithError() throws {
+ try super.setUpWithError()
+
+ let fileManager = FileManager.default
+ let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
+ try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
+ workingDirectoryPath = FilePath(directoryURL.path)
+ }
+
+ func testBasicExample() throws {
+ let docs = [
+ "CNTK formerly known as Computational Network Toolkit",
+ "is a free easy-to-use open-source commercial-grade toolkit",
+ "that enable us to train deep learning algorithms to learn like the human brain."
+ ]
+
+ /// Setting up the environment
+ let env = try Environment()
+ try env.setMapSize(ONE_GB)
+ try env.setMaxReaders(DEFAULT_MAXREADERS)
+ try env.setMaxDBs(DEFAULT_MAXDBS)
+ try env.open(path: workingDirectoryPath)
+
+ /// Writing to LMDB
+ let transaction = try Transaction.begin(.write, in: env)
+
+ var corpus = try DurableHNSWCorpus(
+ encoding: .glove6B50d,
+ namespace: "testBasicExample",
+ in: transaction
+ )
+
+ for doc in docs {
+ try corpus.addUntokenizedDocument(doc, in: transaction)
+ }
+
+ try transaction.commit()
+
+ /// Reading from LMDB
+ let readTransaction = try Transaction.begin(.read, in: env)
+
+ let readCorpus = try DurableHNSWCorpus(
+ encoding: .glove6B50d,
+ namespace: "testBasicExample",
+ in: readTransaction
+ )
+
+ readTransaction.abort()
+
+ // XCTAssert(readCorpus.count == 3)
+ /// readCorpus.count == 3 will fail because we have not fixed the bug with setting size upon reads
+ /// This is because size is only incremented when insertion is called but it is not called when read from disk!
+ }
+
+ func testBasicQueryExample() async throws {
+ let docs = [
+ "The quick brown fox jumps over the lazy dog",
+ "I enjoy taking long walks along the beach at sunset",
+ "Advances in neural networks have enabled new AI capabilities",
+ "The stock market experienced a significant downturn last week",
+ "Cooking a good meal can be both an art and a science",
+ "The exploration of space is both challenging and rewarding",
+ "Machine learning models are becoming increasingly sophisticated",
+ "I love reading about history and ancient civilizations"
+ ]
+
+ let query = "I like to read about new technology and artificial intelligence"
+ let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
+
+ /// Setting up the environment
+ let env = try Environment()
+ try env.setMapSize(ONE_GB)
+ try env.setMaxReaders(DEFAULT_MAXREADERS)
+ try env.setMaxDBs(DEFAULT_MAXDBS)
+ try env.open(path: workingDirectoryPath)
+
+ let transaction = try Transaction.begin(.write, in: env)
+
+ /// Saving the memory map to disk
+ var corpus = try DurableHNSWCorpus(
+ encoder: _documentEncoder,
+ namespace: "testBasicQueryExample",
+ in: transaction
+ )
+
+ for doc in docs {
+ try corpus.addUntokenizedDocument(doc, in: transaction)
+ }
+
+ corpus.saveDictionaryToDownloads(fileName: "dictionary.mmap")
+
+ try transaction.commit()
+
+ do {
+ let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
+
+ /// Reading the memory map (and dictionary) from disk
+ let readTransaction = try Transaction.begin(.write, in: env)
+
+ let readCorpus = try DurableHNSWCorpus(
+ encoder: _documentEncoder,
+ namespace: "testBasicQueryExample",
+ in: readTransaction
+ )
+
+ readCorpus.dictionary = DurableHNSWCorpus.readDictionaryFromDownloads(fileName: "dictionary.mmap") // TODO: move this to initializer?
+
+ let result = try readCorpus.encodedDocuments.find(near: queryVector, limit: 8, in: transaction)
+
+ for result in result {
+ let key = Int(result.id.foreignKey)!
+ print(readCorpus.getUntokenizedDocument(at: key))
+ }
+ } catch {
+ print("Error when trying corpus.encodedDocuments.find(): \(error)")
+ }
+
+ try transaction.commit()
+ }
+
+ func testBuildSubredditCorpus() async throws {
+ /// Generates the LMDB durable storage to disk but runs no tests otherwise
+
+ /// Setting up the environment
+ let env = try Environment()
+ try env.setMapSize(ONE_GB)
+ try env.setMaxReaders(DEFAULT_MAXREADERS)
+ try env.setMaxDBs(DEFAULT_MAXDBS)
+ try env.open(path: workingDirectoryPath)
+
+ /// Get subreddit data
+ guard let submissionsURL = Bundle.module.url(forResource: "Guelph_submissions", withExtension: "zst") else {
+ fatalError("Failed to find waterloo_submissions.zst in test bundle.")
+ }
+ guard let submissionsData = try? Data(contentsOf: submissionsURL) else {
+ fatalError("Failed to load waterloo_submissions.zst from test bundle.")
+ }
+
+ let (submissions, _ ): ([Submission],[Data]) = try await loadFromRedditArchive(submissionsData)
+
+ let transaction = try Transaction.begin(.write, in: env)
+
+ let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
+
+ var corpus = try DurableHNSWCorpus(
+ encoder: _documentEncoder,
+ namespace: "subreddit_durable",
+ in: transaction
+ )
+
+ /// Add documents to corpus
+ for submission in submissions {
+ if let text = submission.selftext {
+ try corpus.addUntokenizedDocument(text, in: transaction)
+ }
+ }
+
+ /// Save dictionary to disk
+ corpus.saveDictionaryToDownloads(fileName: "dictionary.mmap")
+
+ try transaction.commit()
+ }
+
+ func testSubredditQueryExample() async throws {
+ let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
+
+ /// Setting up the environment
+ let env = try Environment()
+ try env.setMapSize(ONE_GB)
+ try env.setMaxReaders(DEFAULT_MAXREADERS)
+ try env.setMaxDBs(DEFAULT_MAXDBS)
+ try env.open(path: workingDirectoryPath)
+
+ /// Reading the memory map (and dictionary) from disk
+ let transaction = try Transaction.begin(.read, in: env)
+
+ let corpus = try DurableHNSWCorpus(
+ encoder: _documentEncoder,
+ namespace: "subreddit_durable",
+ in: transaction
+ )
+
+ corpus.dictionary = DurableHNSWCorpus.readDictionaryFromDownloads(fileName: "dictionary.mmap")
+
+ let query = "I love waterloo and I love the geese."
+ let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
+
+ let result = try corpus.encodedDocuments.find(near: queryVector, limit: 8, in: transaction)
+
+ for result in result {
+ let key = Int(result.id.foreignKey)!
+ print(corpus.getUntokenizedDocument(at: key))
+ }
+ }
+}
+#endif
+
diff --git a/Tests/SwiftNLPTests/2. Encoding/EphemeralHNSWCorpusTests.swift b/Tests/SwiftNLPTests/2. Encoding/EphemeralHNSWCorpusTests.swift
new file mode 100644
index 0000000000000000000000000000000000000000..a4594629ef2da9a54f3599ba9e5ce0f49bab7908
--- /dev/null
+++ b/Tests/SwiftNLPTests/2. Encoding/EphemeralHNSWCorpusTests.swift
@@ -0,0 +1,196 @@
+#if os(macOS)
+import XCTest
+import Foundation
+import System
+@testable import SwiftNLP
+
+final class EphemeralHNSWCorpusTests: XCTestCase {
+ // MARK: There is also an HNSWCorpusDataHandler class which can store an EphemeralHNSWCorpus into a memory map
+ /// However, it is not recommended to use this for large datasets since it uses a currently slow coding protocol conformance with JSONEncoder/Decoder
+
+ // MARK: EphemeralHNSWCorpus can also be used as its typealias HNSWCorpus
+
+ // Load a small set of documents and confirm that corpus and dictionary are updated accordingly
+ func testBasicExample() throws {
+ let docs = [
+ "CNTK formerly known as Computational Network Toolkit",
+ "is a free easy-to-use open-source commercial-grade toolkit",
+ "that enable us to train deep learning algorithms to learn like the human brain."
+ ]
+
+ var corpus = HNSWCorpus(encoding: .glove6B50d)
+ corpus.addUntokenizedDocuments(docs)
+
+ XCTAssert(corpus.count == 3)
+
+ /// Make sure none of our encodings are zero
+ for c in corpus {
+ XCTAssertNotEqual(c, corpus.zeroes)
+ }
+ }
+
+ // Load a bigger set of documents and confirm
+ func testLargeExample() throws {
+ let twentyQuotes = [
+ "Imagination is more important than knowledge. - Albert Einstein",
+ "The greatest enemy of knowledge is not ignorance, it is the illusion of knowledge. - Stephen Hawking",
+ "If I have seen further it is by standing on the shoulders of giants. - Isaac Newton",
+ "The universe is a wondrous place! The faster you create unbreakable code, the faster the universe creates people that can break it. - Richard Feynman",
+ "Science is the belief in the ignorance of experts. - Richard Feynman",
+ "The saddest aspect of life right now is that science gathers knowledge faster than society gathers wisdom. - Isaac Asimov",
+ "Science is the poetry of reality. - Richard Dawkins",
+ "To raise new questions, new possibilities, to regard old problems from a new angle, requires creative imagination and marks real advance in science. - Albert Einstein",
+ "The scientist does not study nature because it is useful; he studies it because he delights in it, and he delights in it because it is beautiful. - Henri Poincaré",
+ "Nothing in life is to be feared, it is only to be understood. Now is the time to understand more, so that we may fear less. - Marie Curie",
+ "An experiment is a question which science poses to Nature, and a measurement is the recording of Nature’s answer. - Max Planck",
+ "If you wish to make an apple pie from scratch, you must first invent the universe. - Carl Sagan",
+ "The function of science fiction is not always to predict the future but sometimes to prevent it. - Frank Herbert",
+ "Science is what we understand well enough to explain to a computer. Art is everything else we do. - Donald Knuth",
+ "In science one tries to tell people, in such a way as to be understood by everyone, something that no one ever knew before. But in poetry, it's the exact opposite. - Paul Dirac",
+ "Science is a way of thinking much more than it is a body of knowledge. - Carl Sagan",
+ "Research is what I’m doing when I don’t know what I’m doing. - Wernher von Braun",
+ "The most beautiful thing we can experience is the mysterious. It is the source of all true art and science. - Albert Einstein",
+ "One, remember to look up at the stars and not down at your feet. Two, never give up work. Work gives you meaning and purpose and life is empty without it. Three, if you are lucky enough to find love, remember it is there and don't throw it away. - Stephen Hawking",
+ "All science is either physics or stamp collecting. - Ernest Rutherford"
+ ]
+
+ var corpus = HNSWCorpus(encoding: .glove6B50d)
+ corpus.addUntokenizedDocuments(twentyQuotes)
+
+ XCTAssertEqual(corpus.count, 20)
+
+ /// Make sure none of our encodings are zero
+ for c in corpus {
+ XCTAssertNotEqual(c, corpus.zeroes)
+ }
+ }
+
+ func testSubreddit() async throws {
+ guard let submissionsURL = Bundle.module.url(forResource: "Guelph_submissions", withExtension: "zst") else {
+ fatalError("Failed to find waterloo_submissions.zst in test bundle.")
+ }
+ guard let submissionsData = try? Data(contentsOf: submissionsURL) else {
+ fatalError("Failed to load waterloo_submissions.zst from test bundle.")
+ }
+
+ let (submissions, _ ): ([Submission],[Data]) = try await loadFromRedditArchive(submissionsData)
+
+ var corpus = HNSWCorpus(encoding: .glove6B50d)
+
+ for submission in submissions {
+ if let text = submission.selftext {
+ corpus.addUntokenizedDocument(text)
+ }
+ }
+
+ XCTAssert(corpus.count == 17999)
+ }
+
+ // Load a small set of documents and confirm that corpus and dictionary are updated accordingly
+ func testBasicQueryExample() async throws {
+ let docs = [
+ "The quick brown fox jumps over the lazy dog",
+ "I enjoy taking long walks along the beach at sunset",
+ "Advances in neural networks have enabled new AI capabilities",
+ "The stock market experienced a significant downturn last week",
+ "Cooking a good meal can be both an art and a science",
+ "The exploration of space is both challenging and rewarding",
+ "Machine learning models are becoming increasingly sophisticated",
+ "I love reading about history and ancient civilizations"
+ ]
+
+ let query = "I like to read about new technology and artificial intelligence"
+
+ let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
+ var corpus = HNSWCorpus(encoder: _documentEncoder)
+ corpus.addUntokenizedDocuments(docs)
+
+ do {
+ let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
+ let results = try corpus.encodedDocuments.find(near: queryVector, limit: 8)
+
+ for result in results {
+ print(corpus.getUntokenizedDocument(at: result.id))
+ }
+ } catch {
+ print("Error when trying corpus.encodedDocuments.find(): \(error)")
+ }
+ }
+
+ func testLargeQueryExample() async throws {
+ let docs = [
+ "Imagination is more important than knowledge. - Albert Einstein",
+ "The greatest enemy of knowledge is not ignorance, it is the illusion of knowledge. - Stephen Hawking",
+ "If I have seen further it is by standing on the shoulders of giants. - Isaac Newton",
+ "The universe is a wondrous place! The faster you create unbreakable code, the faster the universe creates people that can break it. - Richard Feynman",
+ "Science is the belief in the ignorance of experts. - Richard Feynman",
+ "The saddest aspect of life right now is that science gathers knowledge faster than society gathers wisdom. - Isaac Asimov",
+ "Science is the poetry of reality. - Richard Dawkins",
+ "To raise new questions, new possibilities, to regard old problems from a new angle, requires creative imagination and marks real advance in science. - Albert Einstein",
+ "The scientist does not study nature because it is useful; he studies it because he delights in it, and he delights in it because it is beautiful. - Henri Poincaré",
+ "Nothing in life is to be feared, it is only to be understood. Now is the time to understand more, so that we may fear less. - Marie Curie",
+ "An experiment is a question which science poses to Nature, and a measurement is the recording of Nature’s answer. - Max Planck",
+ "If you wish to make an apple pie from scratch, you must first invent the universe. - Carl Sagan",
+ "The function of science fiction is not always to predict the future but sometimes to prevent it. - Frank Herbert",
+ "Science is what we understand well enough to explain to a computer. Art is everything else we do. - Donald Knuth",
+ "In science one tries to tell people, in such a way as to be understood by everyone, something that no one ever knew before. But in poetry, it's the exact opposite. - Paul Dirac",
+ "Science is a way of thinking much more than it is a body of knowledge. - Carl Sagan",
+ "Research is what I’m doing when I don’t know what I’m doing. - Wernher von Braun",
+ "The most beautiful thing we can experience is the mysterious. It is the source of all true art and science. - Albert Einstein",
+ "One, remember to look up at the stars and not down at your feet. Two, never give up work. Work gives you meaning and purpose and life is empty without it. Three, if you are lucky enough to find love, remember it is there and don't throw it away. - Stephen Hawking",
+ "All science is either physics or stamp collecting. - Ernest Rutherford"
+ ]
+
+ let query = "I love Albert Einstein!"
+
+ let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
+ var corpus = HNSWCorpus(encoder: _documentEncoder)
+ corpus.addUntokenizedDocuments(docs)
+
+ do {
+ let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
+ let results = try corpus.encodedDocuments.find(near: queryVector, limit: 8)
+
+ for result in results {
+ print(corpus.getUntokenizedDocument(at: result.id))
+ }
+ } catch {
+ print("Error when trying corpus.encodedDocuments.find(): \(error)")
+ }
+ }
+
+ func testSubredditQueryExample() async throws {
+ guard let submissionsURL = Bundle.module.url(forResource: "Guelph_submissions", withExtension: "zst") else {
+ fatalError("Failed to find waterloo_submissions.zst in test bundle.")
+ }
+ guard let submissionsData = try? Data(contentsOf: submissionsURL) else {
+ fatalError("Failed to load waterloo_submissions.zst from test bundle.")
+ }
+
+ let (submissions, _ ): ([Submission],[Data]) = try await loadFromRedditArchive(submissionsData)
+
+ let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
+ var corpus = HNSWCorpus(encoder: _documentEncoder)
+
+ for submission in submissions {
+ if let text = submission.selftext {
+ corpus.addUntokenizedDocument(text)
+ }
+ }
+
+ let query = "Mr. Goose is a very important figure at the University of Waterloo."
+
+ do {
+ let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
+ let results = try corpus.encodedDocuments.find(near: queryVector, limit: 8)
+
+ for result in results {
+ print(corpus.getUntokenizedDocument(at: result.id))
+ }
+ } catch {
+ print("Error when trying corpus.encodedDocuments.find(): \(error)")
+ }
+ }
+}
+#endif
+
diff --git a/Tests/SwiftNLPTests/2. Encoding/HNSWTests.swift b/Tests/SwiftNLPTests/2. Encoding/HNSWTests.swift
deleted file mode 100644
index aa221933cc63ed9f6dd84b18439cb74174e818c4..0000000000000000000000000000000000000000
--- a/Tests/SwiftNLPTests/2. Encoding/HNSWTests.swift
+++ /dev/null
@@ -1,570 +0,0 @@
-#if os(macOS)
-import XCTest
-import Foundation
-import CoreLMDB
-import CoreLMDBCoders
-import System
-@testable import SwiftNLP
-
-final class HNSWTests: XCTestCase {
- /*
- MARK: To save a memory map HNSWCorpus (example):
-
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
- let corpus = HNSWCorpus<Double>(documentEncoder: _documentEncoder)
- corpus.addUntokenizedDocuments(docs)
- let dataHandler = HNSWCorpusDataHandler(corpus: corpus, resource: "hnsw_testbasicexample")
- dataHandler.saveMemoryMap()
- */
-
- /*
- MARK: To load a memory map HNSWCorpus (example):
-
- let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoding: .glove6B50d, resource: "hnsw_testbasicexample")
- */
-
- // Load a small set of documents and confirm that corpus and dictionary are updated accordingly
- func testBasicExample() throws {
- let docs = [
- "CNTK formerly known as Computational Network Toolkit",
- "is a free easy-to-use open-source commercial-grade toolkit",
- "that enable us to train deep learning algorithms to learn like the human brain."
- ]
-
- let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoding: .glove6B50d, resource: "hnsw_testbasicexample")
-// var corpus = HNSWCorpus(encoding: .glove6B50d)
-// corpus.addUntokenizedDocuments(docs)
-//
- let dataHandler = HNSWCorpusDataHandler(corpus: corpus, resource: "hnsw_testbasicexample")
- let corpusSize = dataHandler.getCorpusSize()
-// let dictionarySize = dataHandler.getDictionarySize(includeKey: false)
-// print("Corpus size: \(corpusSize) bytes")
-// print("Dictionary size: \(dictionarySize) bytes")
-// dataHandler.saveMemoryMap()
-
-
- XCTAssert(corpus.count == 3)
-
- // Make sure none of our encodings are zero
- for c in corpus {
- XCTAssertNotEqual(c, corpus.zeroes)
- }
- }
-
- func testBasicExampleDurable() throws {
- let docs = [
- "CNTK formerly known as Computational Network Toolkit",
- "is a free easy-to-use open-source commercial-grade toolkit",
- "that enable us to train deep learning algorithms to learn like the human brain."
- ]
-
- /// Setting up working directory
- let fileManager = FileManager.default
- let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
- try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
- let filepath = FilePath(directoryURL.path)
-
- /// Setting up the environment
- let env = try Environment()
- try env.setMapSize(1_073_741_824) /// 1 GB
- try env.setMaxReaders(126) /// default
- try env.setMaxDBs(10)
- try env.open(path: filepath)
-
- // Writing to disk
- let transaction = try Transaction.begin(.write, in: env)
-
- var corpus = try DurableHNSWCorpus(
- encoding: .glove6B50d,
- namespace: "testbasicexampledurable",
- in: transaction
- )
-
- for doc in docs {
- try corpus.addUntokenizedDocument(doc, in: transaction)
- }
-
- try transaction.commit()
-
- // Reading from disk
-
- let readTransaction = try Transaction.begin(.read, in: env)
-
- let readCorpus = try DurableHNSWCorpus(
- encoding: .glove6B50d,
- namespace: "testbasicexampledurable",
- in: readTransaction
- )
-
- readTransaction.abort()
-
-// XCTAssert(corpus.count == 3)
- XCTAssert(readCorpus.count == 3)
- }
-
-
- // Load a bigger set of documents and confirm that
- func testBiggerExample() throws {
-
- let twentyQuotes = [
- "Imagination is more important than knowledge. - Albert Einstein",
- "The greatest enemy of knowledge is not ignorance, it is the illusion of knowledge. - Stephen Hawking",
- "If I have seen further it is by standing on the shoulders of giants. - Isaac Newton",
- "The universe is a wondrous place! The faster you create unbreakable code, the faster the universe creates people that can break it. - Richard Feynman",
- "Science is the belief in the ignorance of experts. - Richard Feynman",
- "The saddest aspect of life right now is that science gathers knowledge faster than society gathers wisdom. - Isaac Asimov",
- "Science is the poetry of reality. - Richard Dawkins",
- "To raise new questions, new possibilities, to regard old problems from a new angle, requires creative imagination and marks real advance in science. - Albert Einstein",
- "The scientist does not study nature because it is useful; he studies it because he delights in it, and he delights in it because it is beautiful. - Henri Poincaré",
- "Nothing in life is to be feared, it is only to be understood. Now is the time to understand more, so that we may fear less. - Marie Curie",
- "An experiment is a question which science poses to Nature, and a measurement is the recording of Nature’s answer. - Max Planck",
- "If you wish to make an apple pie from scratch, you must first invent the universe. - Carl Sagan",
- "The function of science fiction is not always to predict the future but sometimes to prevent it. - Frank Herbert",
- "Science is what we understand well enough to explain to a computer. Art is everything else we do. - Donald Knuth",
- "In science one tries to tell people, in such a way as to be understood by everyone, something that no one ever knew before. But in poetry, it's the exact opposite. - Paul Dirac",
- "Science is a way of thinking much more than it is a body of knowledge. - Carl Sagan",
- "Research is what I’m doing when I don’t know what I’m doing. - Wernher von Braun",
- "The most beautiful thing we can experience is the mysterious. It is the source of all true art and science. - Albert Einstein",
- "One, remember to look up at the stars and not down at your feet. Two, never give up work. Work gives you meaning and purpose and life is empty without it. Three, if you are lucky enough to find love, remember it is there and don't throw it away. - Stephen Hawking",
- "All science is either physics or stamp collecting. - Ernest Rutherford"
- ]
-
- var corpus = HNSWCorpus(encoding: .glove6B50d)
- corpus.addUntokenizedDocuments(twentyQuotes)
-
- let dataHandler = HNSWCorpusDataHandler(corpus: corpus, resource: "hnsw_testbiggerexample")
- let corpusSize = dataHandler.getCorpusSize()
- let dictionarySize = dataHandler.getDictionarySize(includeKey: false)
- print("Corpus size: \(corpusSize) bytes")
- print("Dictionary size: \(dictionarySize) bytes")
- dataHandler.saveMemoryMap()
-// let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoding: .glove6B50d, resource: "hnsw_testbiggerexample")
-
- XCTAssertEqual(corpus.count, 20)
-
- // Make sure none of our encodings are zero
- for c in corpus {
- XCTAssertNotEqual(c, corpus.zeroes)
- }
- }
-
- func testSubreddit() async throws {
-//
-// guard let submissionsURL = Bundle.module.url(forResource: "Guelph_submissions", withExtension: "zst") else {
-// fatalError("Failed to find waterloo_submissions.zst in test bundle.")
-// }
-// guard let submissionsData = try? Data(contentsOf: submissionsURL) else {
-// fatalError("Failed to load waterloo_submissions.zst from test bundle.")
-// }
-//
-// let (submissions, _ ): ([Submission],[Data]) = try await loadFromRedditArchive(submissionsData)
-//
-// var corpus = HNSWCorpus(encoding: .glove6B50d)
-//
-// for submission in submissions {
-// if let text = submission.selftext {
-// corpus.addUntokenizedDocument(text)
-// }
-// }
-//
-// let dataHandler = HNSWCorpusDataHandler(corpus: corpus, resource: "hnsw_testsubreddit")
-// let corpusSize = dataHandler.getCorpusSize()
-// let dictionarySize = dataHandler.getDictionarySize(includeKey: false)
-// print("Corpus size: \(corpusSize) bytes")
-// print("Dictionary size: \(dictionarySize) bytes")
-// dataHandler.saveMemoryMap()
-
- //print("Loaded \(corpus.count) documents.")
-// let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoding: .glove6B50d, resource: "hnsw_testsubreddit")
-// XCTAssert(corpus.count == 17999)
- }
-
- func testTypicalNeighborhoodSizeExecutionTime() throws {
-// let range = 0..<100
- let values = [2, 4, 8, 20, 32, 128, 512, 1000]
-
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-
- for typicalNeighborhoodSize in values {
- let start = Date()
-
- let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoding: .glove6B50d, typicalNeighborhoodSize: typicalNeighborhoodSize, resource: "hnsw_testbiggerexample")
-
- let buildend = Date()
- let buildtime = buildend.timeIntervalSince(start)
-
- print("buildtime for typicalNeighborhoodSize = \(typicalNeighborhoodSize): \(buildtime) seconds")
-
- let query = "I like to read about new technology and artificial intelligence"
- let queryVector = _documentEncoder.encodeToken(query)
-
- /// only runtime matters so we do not care about success
- let _ = try? corpus.encodedDocuments.find(near: queryVector, limit: 5)
-
- let searchend = Date()
- let searchtime = searchend.timeIntervalSince(buildend)
-
- print("searchtime for typicalNeighborhoodSize = \(typicalNeighborhoodSize): \(searchtime) seconds")
- }
- }
-
- // Refer to AllMiniLM_sampleTest.swift for reference
- func testBasicQueryExample() async throws {
-
-// let docs = [
-// "The quick brown fox jumps over the lazy dog",
-// "I enjoy taking long walks along the beach at sunset",
-// "Advances in neural networks have enabled new AI capabilities",
-// "The stock market experienced a significant downturn last week",
-// "Cooking a good meal can be both an art and a science",
-// "The exploration of space is both challenging and rewarding",
-// "Machine learning models are becoming increasingly sophisticated",
-// "I love reading about history and ancient civilizations"
-// ]
-
- let query = "I like to read about new technology and artificial intelligence"
-//
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-// var corpus = HNSWCorpus(encoder: _documentEncoder)
-// corpus.addUntokenizedDocuments(docs)
-//
-// let dataHandler = HNSWCorpusDataHandler(corpus: corpus, resource: "hnsw_testbasicqueryexample")
-// let corpusSize = dataHandler.getCorpusSize()
-// let dictionarySize = dataHandler.getDictionarySize(includeKey: false)
-// print("Corpus size: \(corpusSize) bytes")
-// print("Dictionary size: \(dictionarySize) bytes")
-// dataHandler.saveMemoryMap()
- let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoder: _documentEncoder, resource: "hnsw_testbasicqueryexample")
-
- do {
- print("Attempting to query corpus.encodedDocuments.find()...")
-
- let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
- let results = try corpus.encodedDocuments.find(near: queryVector, limit: 8)
-
- for result in results {
- print(corpus.getUntokenizedDocument(at: result.id))
- }
-
- print("Query completed!")
- } catch {
- print("Error when trying corpus.encodedDocuments.find(): \(error)")
- }
- }
-
- func testBasicQueryExampleDurable() async throws {
- let docs = [
- "The quick brown fox jumps over the lazy dog",
- "I enjoy taking long walks along the beach at sunset",
- "Advances in neural networks have enabled new AI capabilities",
- "The stock market experienced a significant downturn last week",
- "Cooking a good meal can be both an art and a science",
- "The exploration of space is both challenging and rewarding",
- "Machine learning models are becoming increasingly sophisticated",
- "I love reading about history and ancient civilizations"
- ]
-
- let query = "I like to read about new technology and artificial intelligence"
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-
- /// Setting up working directory
- let fileManager = FileManager.default
- let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
- try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
- let filepath = FilePath(directoryURL.path)
-
- /// Setting up the environment
- let env = try Environment()
- try env.setMapSize(1_073_741_824) /// 1 GB
- try env.setMaxReaders(126) /// default
- try env.setMaxDBs(10)
- try env.open(path: filepath)
-
- let transaction = try Transaction.begin(.write, in: env)
-
- /// Saving the memory map to disk
- var corpus = try DurableHNSWCorpus(
- encoder: _documentEncoder,
- namespace: "testbasicqueryexampledurable",
- in: transaction
- )
-
- for doc in docs {
- try corpus.addUntokenizedDocument(doc, in: transaction)
- }
-
- corpus.saveDictionaryToDownloads(fileName: "dictionary.mmap")
-
- try transaction.commit()
-
- do {
- print("Attempting to query corpus.encodedDocuments.find()...")
-
- let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
-
- /// Reading the memory map (and dictionary) from disk
- let readTransaction = try Transaction.begin(.write, in: env)
-
- let readCorpus = try DurableHNSWCorpus(
- encoder: _documentEncoder,
- namespace: "testbasicqueryexampledurable",
- in: readTransaction
- )
-
-// readCorpus.dictionary = corpus.getDictionary() // FIXME: don't copy over dictionary
- readCorpus.dictionary = DurableHNSWCorpus.readDictionaryFromDownloads(fileName: "dictionary.mmap")
-
- // do not add documents here!
-
- let result = try readCorpus.encodedDocuments.find(near: queryVector, limit: 8, in: transaction)
-
- for result in result {
- let key = Int(result.id.foreignKey)!
- print(readCorpus.getUntokenizedDocument(at: key))
- }
-
- print("Query completed!")
- } catch {
- print("Error when trying corpus.encodedDocuments.find(): \(error)")
- }
-
- try transaction.commit()
- }
-
- func testLargeQueryExample() async throws {
-
- let docs = [
- "Imagination is more important than knowledge. - Albert Einstein",
- "The greatest enemy of knowledge is not ignorance, it is the illusion of knowledge. - Stephen Hawking",
- "If I have seen further it is by standing on the shoulders of giants. - Isaac Newton",
- "The universe is a wondrous place! The faster you create unbreakable code, the faster the universe creates people that can break it. - Richard Feynman",
- "Science is the belief in the ignorance of experts. - Richard Feynman",
- "The saddest aspect of life right now is that science gathers knowledge faster than society gathers wisdom. - Isaac Asimov",
- "Science is the poetry of reality. - Richard Dawkins",
- "To raise new questions, new possibilities, to regard old problems from a new angle, requires creative imagination and marks real advance in science. - Albert Einstein",
- "The scientist does not study nature because it is useful; he studies it because he delights in it, and he delights in it because it is beautiful. - Henri Poincaré",
- "Nothing in life is to be feared, it is only to be understood. Now is the time to understand more, so that we may fear less. - Marie Curie",
- "An experiment is a question which science poses to Nature, and a measurement is the recording of Nature’s answer. - Max Planck",
- "If you wish to make an apple pie from scratch, you must first invent the universe. - Carl Sagan",
- "The function of science fiction is not always to predict the future but sometimes to prevent it. - Frank Herbert",
- "Science is what we understand well enough to explain to a computer. Art is everything else we do. - Donald Knuth",
- "In science one tries to tell people, in such a way as to be understood by everyone, something that no one ever knew before. But in poetry, it's the exact opposite. - Paul Dirac",
- "Science is a way of thinking much more than it is a body of knowledge. - Carl Sagan",
- "Research is what I’m doing when I don’t know what I’m doing. - Wernher von Braun",
- "The most beautiful thing we can experience is the mysterious. It is the source of all true art and science. - Albert Einstein",
- "One, remember to look up at the stars and not down at your feet. Two, never give up work. Work gives you meaning and purpose and life is empty without it. Three, if you are lucky enough to find love, remember it is there and don't throw it away. - Stephen Hawking",
- "All science is either physics or stamp collecting. - Ernest Rutherford"
- ]
-
- let query = "I love Albert Einstein!"
-
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-// var corpus = HNSWCorpus(encoder: _documentEncoder)
-// corpus.addUntokenizedDocuments(docs)
-//
-// let dataHandler = HNSWCorpusDataHandler(corpus: corpus)
-// let corpusSize = dataHandler.getCorpusSize()
-// let dictionarySize = dataHandler.getDictionarySize(includeKey: false)
-// print("Corpus size: \(corpusSize) bytes")
-// print("Dictionary size: \(dictionarySize) bytes")
-
- let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoder: _documentEncoder, resource: "hnsw_testbiggerexample")
-
- do {
- print("Attempting to query corpus.encodedDocuments.find()...")
-
- let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
- let results = try corpus.encodedDocuments.find(near: queryVector, limit: 8)
-
- for result in results {
- print(corpus.getUntokenizedDocument(at: result.id))
- }
-
- print("Query completed!")
- } catch {
- print("Error when trying corpus.encodedDocuments.find(): \(error)")
- }
- }
-
- func testSubredditQueryExample() async throws {
-
-// guard let submissionsURL = Bundle.module.url(forResource: "Guelph_submissions", withExtension: "zst") else {
-// fatalError("Failed to find waterloo_submissions.zst in test bundle.")
-// }
-// guard let submissionsData = try? Data(contentsOf: submissionsURL) else {
-// fatalError("Failed to load waterloo_submissions.zst from test bundle.")
-// }
-//
-// let (submissions, _ ): ([Submission],[Data]) = try await loadFromRedditArchive(submissionsData)
-//
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-// var corpus = HNSWCorpus(encoder: _documentEncoder)
-//
-// for submission in submissions {
-// if let text = submission.selftext {
-// corpus.addUntokenizedDocument(text)
-// }
-// }
- let corpus = HNSWCorpusDataHandler<Double>.loadMemoryMap(encoder: _documentEncoder, resource: "hnsw_testsubreddit")
-
- let query = "Mr. Goose is a very important figure at the University of Waterloo."
-
- let dataHandler = HNSWCorpusDataHandler(corpus: corpus)
- let corpusSize = dataHandler.getCorpusSize()
- let dictionarySize = dataHandler.getDictionarySize(includeKey: false)
- print("Corpus size: \(corpusSize) bytes")
- print("Dictionary size: \(dictionarySize) bytes")
-
- do {
- print("Attempting to query corpus.encodedDocuments.find()...")
-
- let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
- let results = try corpus.encodedDocuments.find(near: queryVector, limit: 8)
-
- for result in results {
- print(corpus.getUntokenizedDocument(at: result.id))
- }
-
- print("Query completed!")
- } catch {
- print("Error when trying corpus.encodedDocuments.find(): \(error)")
- }
- }
-
- func testGenerateSubredditDurable() async throws {
- /// Generates the LMDB durable storage to disk but runs no tests otherwise
-
- /// Setting up working directory
- let fileManager = FileManager.default
- let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
- try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
- let filepath = FilePath(directoryURL.path)
-
- /// Setting up the environment
- let env = try Environment()
- try env.setMapSize(1_073_741_824) /// 1 GB
- try env.setMaxReaders(126) /// default
- try env.setMaxDBs(10)
- try env.open(path: filepath)
-
- /// Get subreddit data
- guard let submissionsURL = Bundle.module.url(forResource: "Guelph_submissions", withExtension: "zst") else {
- fatalError("Failed to find waterloo_submissions.zst in test bundle.")
- }
- guard let submissionsData = try? Data(contentsOf: submissionsURL) else {
- fatalError("Failed to load waterloo_submissions.zst from test bundle.")
- }
-
- let (submissions, _ ): ([Submission],[Data]) = try await loadFromRedditArchive(submissionsData)
-
- let transaction = try Transaction.begin(.write, in: env)
-
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-
- var corpus = try DurableHNSWCorpus(
- encoder: _documentEncoder,
- namespace: "subreddit_durable",
- in: transaction
- )
-
- /// Add documents to corpus
- for submission in submissions {
- if let text = submission.selftext {
- try corpus.addUntokenizedDocument(text, in: transaction)
- }
- }
-
- /// Save dictionary to disk
- corpus.saveDictionaryToDownloads(fileName: "dictionary.mmap")
-
- try transaction.commit()
- }
-
- func testSubredditDurableQueryExample() async throws {
- /// This test case is just sandbox testing and debugging purposes and is not an actual test case
-
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-
- /// Setting up working directory
- let fileManager = FileManager.default
- let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
- try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
- let filepath = FilePath(directoryURL.path)
-
- /// Setting up the environment
- let env = try Environment()
- try env.setMapSize(1_073_741_824) /// 1 GB
- try env.setMaxReaders(126) /// default
- try env.setMaxDBs(10)
- try env.open(path: filepath)
-
- let transaction = try Transaction.begin(.write, in: env)
- try transaction.commit()
-
- /// Reading the memory map (and dictionary) from disk
- let readTransaction = try Transaction.begin(.write, in: env)
-
- let readCorpus = try DurableHNSWCorpus(
- encoder: _documentEncoder,
- namespace: "subreddit_durable",
- in: readTransaction
- )
-
- readCorpus.dictionary = DurableHNSWCorpus.readDictionaryFromDownloads(fileName: "dictionary.mmap")
-
- let query = "I love waterloo and I love the geese."
- let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
-
- let result = try readCorpus.encodedDocuments.find(near: queryVector, limit: 8, in: transaction)
-
- for result in result {
- let key = Int(result.id.foreignKey)!
- print(readCorpus.getUntokenizedDocument(at: key))
- }
- }
-
- func testAdjustable() async throws {
- /// This test case is just sandbox testing and debugging purposes and is not an actual test case
-
- let _documentEncoder = ContextFreeEncoder<Double>(source: .glove6B50d)
-
- /// Setting up working directory
- let fileManager = FileManager.default
- let directoryURL = fileManager.homeDirectoryForCurrentUser.appendingPathComponent("/Downloads/lmdb")
- try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
- let filepath = FilePath(directoryURL.path)
-
- /// Setting up the environment
- let env = try Environment()
- try env.setMapSize(1_073_741_824) /// 1 GB
- try env.setMaxReaders(126) /// default
- try env.setMaxDBs(10)
- try env.open(path: filepath)
-
- let transaction = try Transaction.begin(.write, in: env)
- try transaction.commit()
-
- /// Reading the memory map (and dictionary) from disk
- let readTransaction = try Transaction.begin(.write, in: env)
-
- let readCorpus = try DurableHNSWCorpus(
- encoder: _documentEncoder,
- namespace: "testbasicqueryexampledurable",
- in: readTransaction
- )
-
- readCorpus.dictionary = DurableHNSWCorpus.readDictionaryFromDownloads(fileName: "dictionary.mmap")
-
- let query = "I like to read about new technology and artificial intelligence"
- let queryVector: [Double] = _documentEncoder.encodeToken(query).map { Double($0) }
-
- let result = try readCorpus.encodedDocuments.find(near: queryVector, limit: 8, in: transaction)
-
- for result in result {
- let key = Int(result.id.foreignKey)!
- print(readCorpus.getUntokenizedDocument(at: key))
- }
- }
-}
-#endif
-