SNLPCorpus + Text Processing.swift

// Copyright (c) 2023 Jim Wallace
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.

import Foundation

extension SNLPCorpus {
    static func applyBasicTextProcessing(_ text: String, characterFilters: CharacterSet, tokenFilters: Set<String>) -> [String] {
        let lowercasedText = text.lowercased()
        
        // Remove characters contained in characterFilters
        let wordsWithoutPunctuationAndNumbers = lowercasedText.components(separatedBy: characterFilters).joined().split(separator: " ")
        
        // Remove tokens contained in tokenFilters
        let wordsWithoutStopWords = wordsWithoutPunctuationAndNumbers.filter { !tokenFilters.contains(String($0)) }
        
        return wordsWithoutStopWords.map { String($0) }
    }
    
    
    static func applyBasicTextProcessing(_ document: [String], characterFilters: CharacterSet, tokenFilters: Set<String>) -> [[String]] {
        return document.map { applyBasicTextProcessing($0, characterFilters: characterFilters, tokenFilters: tokenFilters) }
    }
    
    mutating func addDocument(document: String) {
        let processedDocument = Self.applyBasicTextProcessing(document, characterFilters: CharacterSet.punctuationCharacters, tokenFilters: [])
        addDocument(document: processedDocument)
    }
}