Initial commit

app.py

@@ -0,0 +1,86 @@
+import streamlit as st
+import pandas as pd
+from datasets import Dataset
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import semantic_search
+import torch
+
+model = SentenceTransformer("sentence-transformers/gtr-t5-large")
+
+
+#  Read files
+url = "https://gist.githubusercontent.com/fer-aguirre/b6bdcf59ecae41f84765f72114de9fd1/raw/b4e029fe236c1f38275621686429b2c7aaa3d18b/embeddings.csv"
+
+df_emb = pd.read_csv(url, index_col=0)
+
+df = pd.read_csv('./foia_sample.csv')
+
+dataset = Dataset.from_pandas(df_emb)
+
+dataset_embeddings = torch.from_numpy(dataset.to_pandas().to_numpy()).to(torch.float)
+
+st.markdown("**Inserta una solicitud de información para generar recomendaciones de dependencias**")
+
+if request := st.text_area("", value=""):
+
+    output = model.encode(request)
+
+    query_embeddings = torch.FloatTensor(output)
+
+    hits = semantic_search(query_embeddings, dataset_embeddings, top_k=3)
+
+    id1 = hits[0][0]['corpus_id']
+    id2 = hits[0][1]['corpus_id']
+    id3 = hits[0][2]['corpus_id']
+
+    rec1 = df.iloc[id1].str.split(pat="/")[0]
+    rec2 = df.iloc[id2].str.split(pat="/")[0]
+    rec3 = df.iloc[id3].str.split(pat="/")[0]
+
+    list_rec = [rec1, rec2, rec3]
+    unique_list = []
+    for string in list_rec:
+        if string not in unique_list:
+            unique_list.append(string)
+    st.markdown(f'Recomendaciones:')
+    for rec in unique_list:
+        st.markdown(f':green[{rec[0]}]')
+
+st.markdown("""---""")
+
+if st.button('Genera un ejemplo random'):
+
+    test_example = df['combined'].sample(n=1)
+    index = test_example.index
+    idx = index[0]
+
+    original = df.iloc[idx].str.split(pat="/")[0]
+
+    request = test_example.to_string(index=False)
+
+    st.text(f'{idx}, {request}')
+
+    output = model.encode(request)
+
+    query_embeddings = torch.FloatTensor(output)
+
+    hits = semantic_search(query_embeddings, dataset_embeddings, top_k=3) 
+
+    id1 = hits[0][0]['corpus_id']
+    id2 = hits[0][1]['corpus_id']
+    id3 = hits[0][2]['corpus_id']
+
+    rec1 = df.iloc[id1].str.split(pat="/")[0]
+    rec2 = df.iloc[id2].str.split(pat="/")[0]
+    rec3 = df.iloc[id3].str.split(pat="/")[0]
+
+    list_rec = [rec1, rec2, rec3]
+    unique_list = []
+    for string in list_rec:
+        if string not in unique_list:
+            unique_list.append(string)
+    st.markdown(f'Recomendaciones:')
+    for rec in unique_list:
+        st.markdown(f':green[{rec[0]}]')
+    st.markdown(f'Dependencia original:')
+    st.markdown(f':red[{original[0]}]')

requirements.txt

@@ -0,0 +1,129 @@
+-e git+ssh://git@github.com/fer-aguirre/ai4foia.git@3469e89044d7f0ccfb440fb5762fd7cbd893fa82#egg=AI4FOIA
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==22.2.0
+backcall==0.2.0
+backports.zoneinfo==0.2.1
+blinker==1.5
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+cmake==3.26.1
+comm==0.1.3
+datasets==2.11.0
+debugpy==1.6.6
+decorator==5.1.1
+dill==0.3.6
+distlib==0.3.6
+entrypoints==0.4
+executing==1.2.0
+fastjsonschema==2.16.3
+filelock==3.10.7
+frozenlist==1.3.3
+fsspec==2023.3.0
+gitdb==4.0.10
+GitPython==3.1.31
+huggingface-hub==0.13.3
+idna==3.4
+importlib-metadata==6.1.0
+importlib-resources==5.12.0
+ipykernel==6.22.0
+ipython==8.12.0
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+jupyter_client==8.1.0
+jupyter_core==5.3.0
+lit==16.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+nbformat==5.8.0
+nest-asyncio==1.5.6
+networkx==3.0
+nltk==3.8.1
+numpy==1.24.2
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+packaging==23.0
+pandas==1.5.3
+parso==0.8.3
+pathlib==1.0.1
+pbr==5.11.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.5.0
+pipenv==2023.3.20
+pkgutil_resolve_name==1.3.10
+platformdirs==3.2.0
+prompt-toolkit==3.0.38
+protobuf==3.20.3
+psutil==5.9.4
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==11.0.0
+pydeck==0.8.0
+Pygments==2.14.0
+Pympler==1.0.1
+pyprojroot==0.3.0
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+pyzmq==25.0.2
+regex==2023.3.23
+requests==2.28.2
+responses==0.18.0
+rich==13.3.3
+scikit-learn==1.2.2
+scipy==1.10.1
+semver==2.13.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.97
+six==1.16.0
+smmap==5.0.0
+stack-data==0.6.2
+streamlit==1.20.0
+sympy==1.11.1
+threadpoolctl==3.1.0
+tokenizers==0.13.2
+toml==0.10.2
+toolz==0.12.0
+torch==2.0.0
+torchvision==0.15.1
+tornado==6.2
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.27.4
+triton==2.0.0
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+urllib3==1.26.15
+validators==0.20.0
+virtualenv==20.21.0
+virtualenv-clone==0.5.7
+watchdog==3.0.0
+wcwidth==0.2.6
+xxhash==3.2.0
+yarl==1.8.2
+zipp==3.15.0