data ingestion added

config/config.yaml

@@ -0,0 +1,8 @@
+artifacts_root: artifacts
+
+data_ingestion:
+  root_dir : artifacts/data_ingestion
+  source_dir : E:\\bengla text summarization\BanSum.zip
+  local_data_file : artifacts/data_ingestion/BanSum.zip
+  unzip_dir : artifacts/data_ingestion
+

main.py

@@ -0,0 +1,13 @@
+from src.benglasummarization.logging import logger
+from src.benglasummarization.pipeline.stage01_data_ingestion import DataIngestionPipeline
+
+STAGE_NAME = 'Data Ingestion Stage'
+
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   data_ingestion = DataIngestionPipeline()
+   data_ingestion.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+        logger.exception(e)
+        raise e

params.yaml

@@ -0,0 +1 @@
+key : val

src/benglasummarization/components/data_ingestion.py

@@ -0,0 +1,35 @@
+import os
+import zipfile
+from src.benglasummarization.logging import logger
+from tqdm.notebook import tqdm
+from dataclasses import replace
+from src.benglasummarization.entity.config_entity import DataIngestionConfig
+
+class DataIngestion:
+    def __init__(self, config : DataIngestionConfig):
+        self.config = config
+        
+    def load_file(self):
+        if os.path.exists(self.config.source_dir):
+            self.config = replace(self.config, local_data_file = self.config.source_dir)
+            logger.info(f'File found at: {self.config.local_data_file}')
+        else:
+            logger.info(f'File not found at: {self.config.source_dir}')
+            raise FileNotFoundError(f'No file found at: {self.config.source_dir}')
+        
+    def extract_zip_file(self):
+        unzip_path = self.config.unzip_dir
+        os.makedirs(unzip_path, exist_ok=True)
+        
+        with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:
+            total_files = len(zip_ref.infolist())
+            for file in tqdm(iterable=zip_ref.infolist(), total=total_files, desc = 'Extracting Files'):
+                zip_ref.extract(member = file, path = unzip_path)
+                
+            logger.info(f"Extracted {self.config.local_data_file} to {unzip_path}")
+            
+            
+            
+            
+    
+    

src/benglasummarization/config/configuration.py

@@ -0,0 +1,26 @@
+from src.benglasummarization.constants import *
+from src.benglasummarization.utils.common import read_yaml, create_directories
+from benglasummarization.entity.config_entity import DataIngestionConfig
+class ConfigurationManager:
+    def __init__(
+        self, 
+        config_filepath = CONFIG_FILE_PATH,
+        params_filepath = PARAMS_FILE_PATH
+    ):
+        self.config = read_yaml(config_filepath)
+        self.params = read_yaml(params_filepath)
+        
+        create_directories([self.config.artifacts_root])
+        
+    def get_data_ingestion_config(self) -> DataIngestionConfig:
+        config = self.config.data_ingestion
+        create_directories([config.root_dir])
+        
+        data_ingestion_config = DataIngestionConfig(
+            root_dir= config.root_dir,
+            source_dir=config.source_dir,
+            local_data_file=config.local_data_file,
+            unzip_dir= config.unzip_dir
+        )
+        
+        return data_ingestion_config

src/benglasummarization/entity/config_entity.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+@dataclass(frozen=True)
+class DataIngestionConfig:
+    root_dir : Path
+    source_dir : Path
+    local_data_file : Path
+    unzip_dir : Path
+    

src/benglasummarization/pipeline/stage01_data_ingestion.py

@@ -0,0 +1,14 @@
+from src.benglasummarization.components.data_ingestion import DataIngestion
+from src.benglasummarization.config.configuration import ConfigurationManager
+from src.benglasummarization.config.configuration import ConfigurationManager
+
+class DataIngestionPipeline:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        data_ingestion_config = config.get_data_ingestion_config()
+        data_ingesion = DataIngestion(config=data_ingestion_config)
+        data_ingesion.load_file()
+        data_ingesion.extract_zip_file()