mirror of
https://github.com/hanzoai/datastore-orm.git
synced 2026-09-30 21:40:30 +00:00
zeekay
dda3979851
This is our fork, so it should read like it. `import datastore_orm` instead of `from infi.clickhouse_orm import ...`: `infi` was the original vendor's org namespace, and carrying someone else's org name through every import in the consuming codebase is not a fork, it is a vendored copy that never got renamed. The `infi` namespace package is gone entirely (src/infi/__init__.py with it), the distribution is `datastore-orm`, and the metadata points at Hanzo rather than at the upstream repo and author. NOT renamed, deliberately: the migration history tables are still `infi_clickhouse_orm_migrations`. That name is persisted state — 448 rows in production key off it — so changing it is a data migration, not a rename. Doing it carelessly reproduces the exact failure this fork just hit, where a package rename orphaned the history and silently blocked every migration.
62 lines
1.7 KiB
Python
62 lines
1.7 KiB
Python
import sys
|
|
import nltk
|
|
from nltk.stem.porter import PorterStemmer
|
|
from glob import glob
|
|
from datastore_orm import Database
|
|
from models import Fragment
|
|
|
|
|
|
def trim_punctuation(word):
|
|
'''
|
|
Trim punctuation characters from the beginning and end of the word
|
|
'''
|
|
start = end = len(word)
|
|
for i in range(len(word)):
|
|
if word[i].isalnum():
|
|
start = min(start, i)
|
|
end = i + 1
|
|
return word[start : end]
|
|
|
|
|
|
def parse_file(filename):
|
|
'''
|
|
Parses a text file at the give path.
|
|
Returns a generator of tuples (original_word, stemmed_word)
|
|
The original_word may include punctuation characters.
|
|
'''
|
|
stemmer = PorterStemmer()
|
|
with open(filename, 'r', encoding='utf-8') as f:
|
|
for line in f:
|
|
for word in line.split():
|
|
yield (word, stemmer.stem(trim_punctuation(word)))
|
|
|
|
|
|
def get_fragments(filename):
|
|
'''
|
|
Converts a text file at the given path to a generator
|
|
of Fragment instances.
|
|
'''
|
|
from os import path
|
|
document = path.splitext(path.basename(filename))[0]
|
|
idx = 0
|
|
for word, stem in parse_file(filename):
|
|
idx += 1
|
|
yield Fragment(document=document, idx=idx, word=word, stem=stem)
|
|
print('{} - {} words'.format(filename, idx))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
# Load NLTK data if necessary
|
|
nltk.download('punkt')
|
|
nltk.download('wordnet')
|
|
|
|
# Initialize database
|
|
db = Database('default')
|
|
db.create_table(Fragment)
|
|
|
|
# Load files from the command line or everything under ebooks/
|
|
filenames = sys.argv[1:] or glob('ebooks/*.txt')
|
|
for filename in filenames:
|
|
db.insert(get_fragments(filename), batch_size=100000)
|