Skip to content
Snippets Groups Projects
Commit f7686c69 authored by Jim Wallace's avatar Jim Wallace
Browse files

Added default init() to CoreMLEncoder

parent d78f83b6
No related branches found
No related tags found
1 merge request!15Add interface for using generic CoreML LLMs
Pipeline #116099 failed
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
"kind" : "remoteSourceControl", "kind" : "remoteSourceControl",
"location" : "https://github.com/tannerdsilva/CLMDB.git", "location" : "https://github.com/tannerdsilva/CLMDB.git",
"state" : { "state" : {
"revision" : "39e2f317c898824777365d5669f6fa7571541893", "revision" : "30d45263c8e512b01ab073e77ebff09039bdc593",
"version" : "0.9.31" "version" : "0.9.31"
} }
}, },
...@@ -45,6 +45,15 @@ ...@@ -45,6 +45,15 @@
"version" : "0.1.14" "version" : "0.1.14"
} }
}, },
{
"identity" : "surge",
"kind" : "remoteSourceControl",
"location" : "https://github.com/Jounce/Surge.git",
"state" : {
"revision" : "6e4a47e63da8801afe6188cf039e9f04eb577721",
"version" : "2.3.2"
}
},
{ {
"identity" : "swift-argument-parser", "identity" : "swift-argument-parser",
"kind" : "remoteSourceControl", "kind" : "remoteSourceControl",
......
...@@ -39,9 +39,19 @@ public macro MODEL_VALIDATE_NAME_AND_SET_INPUT_SIZE() = #externalMacro( ...@@ -39,9 +39,19 @@ public macro MODEL_VALIDATE_NAME_AND_SET_INPUT_SIZE() = #externalMacro(
class CoreMLEncoder<Scalar: BinaryFloatingPoint & Codable>: SNLPEncoder { class CoreMLEncoder<Scalar: BinaryFloatingPoint & Codable>: SNLPEncoder {
var zeroes: [Scalar] var zeroes: [Scalar]
var dimensions: UInt
var model: String var model: String
required init() {
zeroes = []
dimensions = 0
model = "all-MiniLM-L6-v2"
}
func encodeToken(_ token: String) -> [Scalar] { func encodeToken(_ token: String) -> [Scalar] {
let tokenization = LLMEmbeddings(model_type: self.model).tokenizer.tokenizeToIds(text: token) as! [Scalar] let tokenization = LLMEmbeddings(model_type: self.model).tokenizer.tokenizeToIds(text: token) as! [Scalar]
return tokenization return tokenization
......
...@@ -34,7 +34,7 @@ final class BERT_test: XCTestCase { ...@@ -34,7 +34,7 @@ final class BERT_test: XCTestCase {
var query_embedding: [Float] = [] var query_embedding: [Float] = []
var embedding_dim: Int = 384 var embedding_dim: Int = 384
var model = LLMEmbeddings(model_type: "gte-small") var model = LLMEmbeddings(model_type: model)
query_embedding = await model.encode(sentence: query[0])! query_embedding = await model.encode(sentence: query[0])!
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment