Skip to content
Snippets Groups Projects
Commit 45edd067 authored by Jim Wallace's avatar Jim Wallace
Browse files

Added NaturalLanguageEncoder sketch

parent 17f46bc9
No related branches found
No related tags found
1 merge request!7Added NaturalLanguageEncoder sketch
Pipeline #108814 passed
......@@ -31,5 +31,6 @@ protocol SNLPEncoder : Codable {
var defaultTokenEncoding: Value { get }
@inlinable
func encode(token: Key) -> Value
}
// Copyright (c) 2024 Henry Tian, Jim Wallace
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
#if canImport(NatualLanguage)
import NaturalLanguage
class NaturalLanguageEncoder<T>: SNLPEncoder where T: Collection & Codable, T.Element: BinaryFloatingPoint {
typealias Key = String
typealias Value = [Float]
var defaultTokenEncoding: [Float]
@inlinable
func encode(token: String) -> [Float] {
if let embedding = NLEmbedding.wordEmbedding(for: .english) {
return embedding
}
return defaultTokenEncoding
}
}
#endif
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment