Initual commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+full_model_v4.pth filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,173 @@
+import streamlit as st
+import json
+import numpy as np
+import torch
+from transformers import (
+    DebertaV2Config,
+    DebertaV2Model,
+    DebertaV2Tokenizer,
+)
+import sentencepiece
+
+model_name = "microsoft/deberta-v3-base"
+tokenizer = DebertaV2Tokenizer.from_pretrained(model_name)
+
+def preprocess_text(text, tokenizer, max_length=512):
+    inputs = tokenizer(
+        text,
+        padding="max_length",
+        truncation=True,
+        max_length=max_length,
+        return_tensors="pt"
+    )
+    return inputs
+
+
+def classify_text(text, model, tokenizer, device, threshold=0.5):
+    inputs = preprocess_text(text, tokenizer)
+    input_ids = inputs["input_ids"].to(device)
+    attention_mask = inputs["attention_mask"].to(device)
+    model.eval()
+    with torch.no_grad():
+        logits = model(input_ids, attention_mask)
+    probs = torch.sigmoid(logits)
+    predictions = (probs > threshold).int().cpu().numpy()
+    
+    return probs.cpu().numpy(), predictions
+
+def get_themes(text, model, tokenizer, label_to_theme, device, limit=5):
+    probabilities, _ = classify_text(text, model, tokenizer, device)
+    themes = []
+    for label in probabilities[0].argsort()[-limit:]:
+        themes.append((label_to_theme[str(label)], probabilities[0][label]))
+    return themes
+
+class DebertPaperClassifier(torch.nn.Module):
+    def __init__(self, num_labels, device, dropout_rate=0.1, class_weights=None):
+        super().__init__()
+        self.config = DebertaV2Config.from_pretrained(model_name)
+        self.deberta = DebertaV2Model.from_pretrained(model_name, config=self.config)
+
+        self.classifier = torch.nn.Sequential(
+            torch.nn.Dropout(dropout_rate),
+            torch.nn.Linear(self.config.hidden_size, 512),
+            torch.nn.LayerNorm(512),
+            torch.nn.GELU(),
+            torch.nn.Dropout(dropout_rate),
+            torch.nn.Linear(512, num_labels)
+        )
+
+        self._init_weights()
+        if class_weights is not None:
+            self.loss_fct = torch.nn.BCEWithLogitsLoss(weight=class_weights.to(device))
+        else:
+            self.loss_fct = torch.nn.BCEWithLogitsLoss()
+
+class DebertPaperClassifierV5(torch.nn.Module):
+    def __init__(self, device, num_labels=47, dropout_rate=0.1, class_weights=None):
+        super().__init__()
+        self.config = DebertaV2Config.from_pretrained("microsoft/deberta-v3-base")
+        self.deberta = DebertaV2Model.from_pretrained("microsoft/deberta-v3-base", config=self.config)
+        
+        self.classifier = torch.nn.Sequential(
+            torch.nn.Dropout(dropout_rate),
+            torch.nn.Linear(self.config.hidden_size, 512),
+            torch.nn.LayerNorm(512),
+            torch.nn.GELU(),
+            torch.nn.Dropout(dropout_rate),
+            torch.nn.Linear(512, num_labels)
+        )
+        
+        if class_weights is not None:
+            self.loss_fct = torch.nn.BCEWithLogitsLoss(weight=class_weights.to(device))
+        else:
+            self.loss_fct = torch.nn.BCEWithLogitsLoss()
+
+    def forward(self, input_ids, attention_mask, labels=None):
+        outputs = self.deberta(
+            input_ids=input_ids,
+            attention_mask=attention_mask
+        )
+        logits = self.classifier(outputs.last_hidden_state[:, 0, :])
+        loss = None
+        if labels is not None:
+            loss = self.loss_fct(logits, labels)
+        return (loss, logits) if loss is not None else logits
+
+    def _init_weights(self):
+        for module in self.classifier.modules():
+            if isinstance(module, torch.nn.Linear):
+                module.weight.data.normal_(mean=0.0, std=0.02)
+                if module.bias is not None:
+                    module.bias.data.zero_()
+
+def forward(self, 
+                input_ids, 
+                attention_mask,
+                labels=None,
+               ):
+        outputs = self.deberta(
+            input_ids=input_ids,
+            attention_mask=attention_mask
+        )
+
+        cls_output = outputs.last_hidden_state[:, 0, :]
+        logits = self.classifier(cls_output)
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_fct(logits, labels)
+
+        return (loss, logits) if loss is not None else logits
+    
+@st.cache_resource
+def load_model(test=False):
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    try:
+        path = 'Homework4'
+        with open(f'{path}/label_to_theme.json', 'r') as f:
+            label_to_theme = json.load(f)
+    except:
+        path = '/Users/bvd757/my_documents/Машинное обучение 2/Howework4'
+        with open(f'{path}/label_to_theme.json', 'r') as f:
+            label_to_theme = json.load(f)
+
+    class_weights = torch.load(f'{path}/class_weights.pth').to(device)
+
+    model = DebertPaperClassifier(device=device, num_labels=len(label_to_theme), class_weights=class_weights).to(device)
+    model.load_state_dict(torch.load(f"{path}/full_model_v4.pth", map_location=device))
+    if test:
+        print(device)
+        print(model)
+        print("Model!!!")
+        text = 'We propose an architecture for VQA which utilizes recurrent layers to\ngenerate visual and textual attention. The memory characteristic of the\nproposed recurrent attention units offers a rich joint embedding of visual and\ntextual features and enables the model to reason relations between several\nparts of the image and question. Our single model outperforms the first place\nwinner on the VQA 1.0 dataset, performs within margin to the current\nstate-of-the-art ensemble model. We also experiment with replacing attention\nmechanisms in other state-of-the-art models with our implementation and show\nincreased accuracy. In both cases, our recurrent attention mechanism improves\nperformance in tasks requiring sequential or relational reasoning on the VQA\ndataset.'
+        print(get_themes(text, model, tokenizer, label_to_theme, device))
+    return model, tokenizer, label_to_theme, device
+
+def kek():
+    
+    title = st.text_input("Title")
+    abstract = st.text_area("Abstract")
+    
+    if st.button("Classify"):
+        if not title and not abstract:
+            st.warning("Please enter an abstract")
+            return
+        
+        text = f"{title}\n\n{abstract}" if title and abstract else title or abstract
+        model, tokenizer, label_to_theme, device = load_model()
+        
+        with st.spinner("Classifying..."):
+            themes = get_themes(text, model, tokenizer, label_to_theme, device, lim)
+        
+        st.success("Classification results:")
+        for theme, prob in themes:
+            st.write(f"- {theme}: {prob:.2%}")
+
+
+if __name__ == "__main__":
+    inp = '0'
+    if inp != '0':
+        kek()
+    else:
+        load_model(True)

full_model_v4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a8bd37371fde6f8e3c74815a6393577db39a188aca551475c862d70c67c98b3
+size 737088702

label_to_theme.json

@@ -0,0 +1 @@
+{"0": "cs.AI", "1": "physics.soc-ph", "2": "stat.ML", "3": "cs.CE", "4": "cs.DB", "5": "cs.CL", "6": "cs.NA", "7": "cs.CY", "8": "cs.GT", "9": "cs.SI", "10": "stat.AP", "11": "cs.DL", "12": "math.ST", "13": "nlin.AO", "14": "cs.LO", "15": "cs.MM", "16": "cond-mat.dis-nn", "17": "cs.DM", "18": "cs.CC", "19": "stat.CO", "20": "cs.DC", "21": "cs.IT", "22": "cs.DS", "23": "cs.SY", "24": "q-bio.QM", "25": "cs.PL", "26": "cs.RO", "27": "cs.NE", "28": "cs.CR", "29": "cs.MA", "30": "q-bio.NC", "31": "cs.LG", "32": "cs.GR", "33": "physics.data-an", "34": "quant-ph", "35": "cs.IR", "36": "math.NA", "37": "math.PR", "38": "stat.ME", "39": "cs.SE", "40": "math.OC", "41": "math.IT", "42": "cs.HC", "43": "stat.TH", "44": "cs.NI", "45": "cs.CV", "46": "cs.SD"}

requirements.txt

@@ -0,0 +1,121 @@
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.14
+aiosignal==1.3.2
+altair==5.5.0
+annotated-types==0.7.0
+appnope==0.1.4
+asttokens==3.0.0
+async-timeout==5.0.1
+attrs==25.3.0
+blinker==1.9.0
+cachetools==5.5.2
+certifi==2025.1.31
+charset-normalizer==3.4.1
+click==8.1.8
+colored==2.3.0
+comm==0.2.2
+contourpy==1.3.0
+cycler==0.12.1
+datasets==3.5.0
+debugpy==1.8.12
+decorator==5.2.1
+dill==0.3.8
+distlib==0.3.9
+docker-pycreds==0.4.0
+einops==0.8.1
+eval_type_backport==0.2.2
+exceptiongroup==1.2.2
+executing==2.2.0
+filelock==3.17.0
+fonttools==4.56.0
+frozenlist==1.5.0
+fsspec==2024.12.0
+gitdb==4.0.12
+GitPython==3.1.44
+huggingface-hub==0.29.3
+idna==3.10
+imageio==2.37.0
+importlib_metadata==8.6.1
+importlib_resources==6.5.2
+ipykernel==6.29.5
+ipython==8.18.1
+jedi==0.19.2
+Jinja2==3.1.5
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kiwisolver==1.4.7
+lazy_loader==0.4
+MarkupSafe==3.0.2
+matplotlib==3.9.4
+matplotlib-inline==0.1.7
+mpmath==1.3.0
+multidict==6.2.0
+multiprocess==0.70.16
+narwhals==1.33.0
+nest-asyncio==1.6.0
+networkx==3.2.1
+numpy==2.0.2
+opencv-python==4.11.0.86
+packaging==24.2
+pandas==2.2.3
+parso==0.8.4
+pexpect==4.9.0
+pillow==11.1.0
+platformdirs==4.3.6
+prompt_toolkit==3.0.50
+propcache==0.3.1
+protobuf==5.29.4
+psutil==7.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyarrow==19.0.1
+pydantic==2.11.1
+pydantic_core==2.33.0
+pydeck==0.9.1
+Pygments==2.19.1
+pyparsing==3.2.1
+python-dateutil==2.9.0.post0
+pytz==2025.1
+PyYAML==6.0.2
+pyzmq==26.2.1
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+rpds-py==0.24.0
+safetensors==0.5.3
+scikit-image==0.24.0
+scikit-learn==1.6.1
+scipy==1.13.1
+seaborn==0.13.2
+sentencepiece==0.2.0
+sentry-sdk==2.24.1
+setproctitle==1.3.5
+smmap==5.0.2
+stack-data==0.6.3
+streamlit==1.44.1
+sympy==1.13.1
+tenacity==9.1.2
+termcolor==2.5.0
+threadpoolctl==3.5.0
+tifffile==2024.8.30
+tokenizers==0.21.1
+toml==0.10.2
+torch==2.2.0
+torchvision==0.17.0
+tornado==6.4.2
+tqdm==4.67.1
+traitlets==5.14.3
+transformers==4.50.3
+typing-inspection==0.4.0
+typing_extensions==4.12.2
+tzdata==2025.1
+urllib3==2.3.0
+virtualenv==20.30.0
+wandb==0.19.8
+wcwidth==0.2.13
+xxhash==3.5.0
+yarl==1.18.3
+zipp==3.21.0