Remove openai_vectorize calls from crawl function and retain vectorize for article processing
Browse files- source/cbirc.py +1 -2
source/cbirc.py
CHANGED
@@ -13,7 +13,7 @@ from controllers.utils import (
|
|
13 |
translate,
|
14 |
update_content,
|
15 |
)
|
16 |
-
from controllers.vectorizer import
|
17 |
|
18 |
@task(name = "Data Collection - cbirc", log_prints = True)
|
19 |
def crawl(delta):
|
@@ -78,6 +78,5 @@ def crawl(delta):
|
|
78 |
article['referenceid'] = None
|
79 |
update_content(article)
|
80 |
vectorize(article)
|
81 |
-
openai_vectorize(article)
|
82 |
except (ValueError, KeyError, TypeError) as error:
|
83 |
logger.error(error)
|
|
|
13 |
translate,
|
14 |
update_content,
|
15 |
)
|
16 |
+
from controllers.vectorizer import vectorize
|
17 |
|
18 |
@task(name = "Data Collection - cbirc", log_prints = True)
|
19 |
def crawl(delta):
|
|
|
78 |
article['referenceid'] = None
|
79 |
update_content(article)
|
80 |
vectorize(article)
|
|
|
81 |
except (ValueError, KeyError, TypeError) as error:
|
82 |
logger.error(error)
|