m. polinsky
commited on
Commit
·
63dcf6f
1
Parent(s):
e790551
Checking digestion
Browse files- digestor.py +3 -0
digestor.py
CHANGED
|
@@ -72,9 +72,11 @@ class Digestor:
|
|
| 72 |
# Loop through stubs, collecting data and instantiating
|
| 73 |
# and collecting Summary objects.
|
| 74 |
for stub in self.stubs:
|
|
|
|
| 75 |
# Check to see if we already have access to this summary:
|
| 76 |
if not isinstance(stub, stb):
|
| 77 |
self.summaries.append(stub)
|
|
|
|
| 78 |
else:
|
| 79 |
# if not:
|
| 80 |
summary_data: List
|
|
@@ -86,6 +88,7 @@ class Digestor:
|
|
| 86 |
# Start chunk timer
|
| 87 |
with Timer(name=f"{stub.hed}_chunk_time", logger=None):
|
| 88 |
chunk_list = self.chunk_piece(text, self.word_limit, stub.source.source_summarization_checkpoint)
|
|
|
|
| 89 |
# start totoal summarization timer. Summarization queries are timed in 'perform_summarzation()'
|
| 90 |
with Timer(name=f"{stub.hed}_summary_time", text="Whole article summarization time: {:.4f} seconds"):
|
| 91 |
summary = self.perform_summarization(
|
|
|
|
| 72 |
# Loop through stubs, collecting data and instantiating
|
| 73 |
# and collecting Summary objects.
|
| 74 |
for stub in self.stubs:
|
| 75 |
+
|
| 76 |
# Check to see if we already have access to this summary:
|
| 77 |
if not isinstance(stub, stb):
|
| 78 |
self.summaries.append(stub)
|
| 79 |
+
print(f"""type(stub): {type(stub)}""")
|
| 80 |
else:
|
| 81 |
# if not:
|
| 82 |
summary_data: List
|
|
|
|
| 88 |
# Start chunk timer
|
| 89 |
with Timer(name=f"{stub.hed}_chunk_time", logger=None):
|
| 90 |
chunk_list = self.chunk_piece(text, self.word_limit, stub.source.source_summarization_checkpoint)
|
| 91 |
+
print(f"""\n\n=============\nchunkList: {chunk_list}\n===========================\n\n""")
|
| 92 |
# start totoal summarization timer. Summarization queries are timed in 'perform_summarzation()'
|
| 93 |
with Timer(name=f"{stub.hed}_summary_time", text="Whole article summarization time: {:.4f} seconds"):
|
| 94 |
summary = self.perform_summarization(
|