update blooms level, token usage, parallel run

.gitignore

@@ -141,6 +141,8 @@ dmypy.json
 
 logs/
 src/task_generation/init_book_chapter_text_files
+src/task_generation/init_finance_book_chapter_text_files
+src/task_generation/init_math_book_chapter_text_files
 src/task_generation/other_scripts
 src/outputs/
 outputs/

src/base_stages/stage3_tasks.py

@@ -51,6 +51,8 @@ def run_stage3(
     experiment_id = cfg.exp_cfg.exp_id
     output_base_dir = Path(cfg.global_cfg.output_dir)
     task_gen_mode = str(cfg.task_generation_cfg.get("mode", "base")).strip().lower()
+    worker_index = cfg.task_generation_cfg.get("worker_index", None)
+    worker_count = cfg.task_generation_cfg.get("worker_count", None)
 
     # Determine tasks tag (resume or new)
     is_resume = tasks_tag is not None
@@ -62,12 +64,19 @@ def run_stage3(
     # If agentic mode, delegate to runner module which will call back into this module.
     if task_gen_mode == "agentic":
         logger.info("Stage 3 mode: agentic")
+        if worker_index is not None or worker_count is not None:
+            logger.info(
+                "Agentic Stage 3 sharding enabled: worker_index=%s, worker_count=%s",
+                worker_index,
+                worker_count,
+            )
         return run_from_stage3(
             experiment_id=experiment_id,
             output_base_dir=output_base_dir,
             capabilities_tag=capabilities_tag,
             tasks_tag=tasks_tag,
-            is_resume=is_resume,
+            worker_index=int(worker_index) if worker_index is not None else None,
+            worker_count=int(worker_count) if worker_count is not None else None,
         )
 
     if task_gen_mode != "base":

src/cfg/run_cfg.yaml

@@ -71,6 +71,8 @@ task_generation_cfg:
   tasks_per_blueprint: 1
   min_subtopics: 1
   max_subtopics: 1
+  worker_index: null
+  worker_count: null
 
 # =============================================================================
 # EVALUATION PIPELINE

src/cfg/task_generation/agent_config.yaml

@@ -16,7 +16,7 @@ agents:
           api_key: "${GOOGLE_API_KEY}"
       temperature: 0
       top_p: 0.95
-      timeout: 1200
+      timeout: 1800
       cache_seed: 42
 
   verifier:
@@ -28,11 +28,11 @@ agents:
           api_key: "${ANTHROPIC_API_KEY}"
       temperature: 0
       top_p: 0.95
-      timeout: 600
+      timeout: 1500
       cache_seed: 42
 
 dedup:
-  enabled: false
+  enabled: true
   embedding_model: "text-embedding-3-small"
   threshold: 0.90
   keep_policy: "first"

src/cfg/task_generation/pipeline_config.yaml

@@ -8,7 +8,7 @@ pipeline:
 
   # Chapter corpus root (relative to repo root)
   # runner expects: <book_chapter_dir>/**.txt
-  book_chapter_dir: "init_book_chapter_text_files"
+  book_chapter_dir: "init_finance_book_chapter_text_files"
 
   # Blueprints file (relative to cfg/)
   blueprints_file: "blueprints.json"
@@ -26,11 +26,10 @@ pipeline:
 
   # Loop constraints
   max_retries: 3
-  num_tasks: 50
-  hardening_rounds: 5
+  hardening_rounds: 0
 
   # Default tasks per combo (can be overridden by each combo's "num_tasks" in blueprints.json)
-  num_tasks_per_combo: 15
+  num_tasks_per_combo: 10
 
   # Checkpoint configuration for incremental save/resume within a chapter run
   checkpoint:

src/schemas/task_gen_io_utils.py

@@ -6,8 +6,11 @@
 from typing import Any, Sequence
 
 
+_TextMessage: type[Any] | None
 try:
-    from autogen_agentchat.messages import TextMessage as _TextMessage
+    from autogen_agentchat.messages import TextMessage as _ImportedTextMessage
+
+    _TextMessage = _ImportedTextMessage
 except ModuleNotFoundError:  # pragma: no cover - optional runtime dependency
     _TextMessage = None