Initial Pr (#1)

Initial Pr

Author: npatta01

.dockerignore

@@ -0,0 +1,6 @@
+data/
+workshop_infra/
+!workshop_infra/scripts/
+
+
+

.gitignore

@@ -127,3 +127,18 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+
+data/
+
+
+workshop_infra/cert/*
+workshop_infra/config.yaml
+workshop_infra/key_file.json
+
+*.db
+tmp/
+.DS_Store
+
+!/**/.gitkeep
+workshop_infra/keyfile.json

Dockerfile

@@ -0,0 +1,50 @@
+
+#FROM jupyter/scipy-notebook:python-3.10.6
+FROM jupyter/scipy-notebook:python-3.7.12
+
+
+
+USER root
+
+
+RUN apt-get update && apt-get --yes install apt-utils && \
+    apt-get --yes --no-install-recommends install htop tmux graphviz curl build-essential libsasl2-dev gfortran && \
+    apt-get clean;
+
+
+# set the user back to original setting
+USER $NB_UID
+
+
+
+# Install from requirements.txt file
+COPY --chown=${NB_UID}:${NB_GID} environment.yaml /tmp/
+
+RUN mamba env update -n base -f /tmp/environment.yaml && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+COPY --chown=${NB_UID}:${NB_GID} workshop_infra/scripts /tmp/scripts/
+
+USER root
+
+RUN bash /tmp/scripts/build_setup_root.sh
+USER $NB_UID
+
+
+COPY --chown=${NB_UID}:${NB_GID} notebooks/workshop_setup.ipynb /tmp/workshop/notebooks/
+
+RUN bash /tmp/scripts/build_setup_user.sh
+
+COPY --chown=${NB_UID}:${NB_GID} . /tmp/workshop/
+
+ENV PATH="/opt/google-cloud-sdk/bin:${PATH}" 
+
+
+#COPY --chown=${NB_UID}:${NB_GID} docker-setup.sh /tmp/
+
+#COPY --chown=${NB_UID}:${NB_GID} setup.ipynb /tmp/
+
+# RUN papermill /tmp/setup.ipynb /tmp/setup__out.ipynb -k python3 --log-output --log-level INFO --progress-bar && \
+#     fix-permissions "${CONDA_DIR}" && \
+#     fix-permissions "/home/${NB_USER}"

assets/all_assets.sw

@@ -0,0 +1,51 @@
+version: "3.0"
+services:
+  elasticsearch:
+    container_name: es-container
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.4.3
+    environment:
+      - xpack.security.enabled=false
+      - "discovery.type=single-node"
+    ports:
+      - 9200:9200
+    volumes:
+        - esdata:/usr/share/elasticsearch/data
+        
+#   milvus:
+#     container_name: milvus
+#     image: milvusdb/milvus:1.1.1-cpu-d061621-330cc6
+#     ports:
+#       - 19530:19530 
+#       - 19121:19121
+#     volumes:
+#         - milvusdata:/var/lib/milvus
+
+
+  milvus:
+    container_name: milvus
+    build:
+      context: docker_milvus    
+    ports:
+      - 19530:19530 
+      - 19121:19121
+    volumes:
+        - milvusdata:/var/lib/milvus
+ 
+        
+  weaviate:
+    image: semitechnologies/weaviate:1.14.0
+    ports:
+      - 8081:8080
+    environment:
+        QUERY_DEFAULTS_LIMIT: 25
+        AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+        PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+        DEFAULT_VECTORIZER_MODULE: 'none'
+        ENABLE_MODULES: ''
+        CLUSTER_HOSTNAME: 'node1'  
+    volumes:
+      - weaviatedata:/var/lib/weaviate
+volumes:
+  esdata:
+  weaviatedata:
+  milvusdata:

data/.gitkeep

@@ -0,0 +1,26 @@
+FROM ubuntu:18.04 
+
+ENV MILVUS_VERSION "2.1.4-1"
+
+COPY install.sh /tmp/install.sh
+
+RUN bash /tmp/install.sh
+
+
+# ARG S6_OVERLAY_VERSION=3.1.2.1
+
+
+# ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp
+
+# RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz
+
+# ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-x86_64.tar.xz /tmp
+
+# RUN tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz
+
+
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+
+
+CMD ["/usr/bin/supervisord"]

docker-compose.yaml

@@ -0,0 +1,26 @@
+apt-get update -y 
+
+apt install software-properties-common -y
+
+add-apt-repository ppa:milvusdb/milvus
+
+
+apt-get update -y 
+
+
+apt-get install "milvus=$MILVUS_VERSION"  -y
+
+
+
+#mkdir -p /etc/services.d/system/
+
+#cp /lib/systemd/system/milvus* /etc/services.d/system/
+
+#cp /lib/systemd/system/milvus* /etc/services.d/system/
+
+#COPY resources/docker/services.d /etc/services.d
+
+
+
+apt-get update && apt-get install -y supervisor
+mkdir -p /var/log/supervisor

docker_milvus/Dockerfile

@@ -0,0 +1,21 @@
+https://github.com/just-containers/s6-overlay
+
+
+
+cat /etc/services.d/system/milvus-etcd.service 
+ExecStart=/usr/bin/milvus-etcd --data-dir /var/lib/milvus/etcd-data
+
+
+cat /etc/services.d/system/milvus-minio.service 
+ExecStart=/usr/bin/milvus-minio server /var/lib/milvus/minio-data
+
+
+
+cat /etc/services.d/system/milvus.service 
+
+Environment=MILVUSCONF=/etc/milvus/configs/
+ExecStart=/usr/bin/milvus run standalone
+
+
+
+https://gdevillele.github.io/engine/admin/using_supervisord/

docker_milvus/install.sh

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=true
+
+[program:milvus-minio]
+command=/usr/bin/milvus-minio server /var/lib/milvus/minio-data
+
+[program:milvus-etcd]
+command=/usr/bin/milvus-etcd --data-dir /var/lib/milvus/etcd-data
+
+
+[program:milvus]
+environment=MILVUSCONF=/etc/milvus/configs/
+command=/usr/bin/milvus run standalone

docker_milvus/readme.md

@@ -0,0 +1,42 @@
+# Internal Notes
+
+## Setup Dep
+
+Install other deps
+
+```bash
+sudo apt update && sudo apt install -y p7zip-full
+```
+
+Create conda environemnt
+
+```bash
+conda create -n stackoverflow python=3.7 mamba
+conda activate stackoverflow
+mamba env update -n stackoverflow -f environment.yaml 
+# mamba install anaconda jupyter ipykernel nb_conda_kernels
+
+mamba install ipython ipykernel nb_conda_kernels
+
+ipython kernel install --user --name=stackoverflow
+
+
+conda create --name stackoverflow --clone base
+
+```
+
+Start ES/ Faiss for local dev
+
+```bash
+docker-compose up
+```
+
+```bash
+docker run --user root -e GRANT_SUDO=yes -it app bash
+```
+
+
+
+```
+
+```

docker_milvus/supervisord.conf

@@ -0,0 +1,85 @@
+PUT /items
+
+```json
+{
+  "mappings": {
+    "properties": {
+      "title":          { "type": "text"    },
+      "description":    { "type": "text"    },
+
+      "brand":          { "type": "keyword" },
+      "product_type":   { "type": "keyword" },
+
+      "price":          { "type": "double"  }
+    }
+  }
+}
+```
+
+Nike shoe under 100$ 
+
+GET /items/_search
+
+```json
+{
+  "query": {
+    
+    "multi_match": {
+        "query": "Nike shoe under 100$",
+        "fields": ["title^2", "Description^1"]
+    }
+
+    ,"bool": {
+      "filter": [
+        { "term": { "brand": "nike" }}
+      ]
+    }
+    ,"filtered": {           
+        "filter": {
+            "range":  {
+                "price" : {  "lte": 100 }
+            }
+        }
+    }   
+}
+
+```
+
+
+
+## PR curve
+```
+Recall	Perfect Classifier	Baseline Classifier	Good Classifier	High Precision
+0.1	0.95	0.5	0.9	0.91
+0.2	0.95	0.5	0.85	0.91
+0.3	0.95	0.5	0.85	0.91
+0.4	0.95	0.5	0.8	0.9
+0.5	0.95	0.5	0.8	0.4
+0.6	0.95	0.5	0.8	0.4
+0.7	0.95	0.5	0.8	0.4
+0.8	0.95	0.5	0.8	0.2
+0.9	0.95	0.5	0.7	0.2
+1	0.9	0.5	0.2	0.1
+```
+
+
+
+
+dcg
+
+```
+Discounted\space Cumulative\space Gain
+ = \sum_{1}^{p}\frac{ relevance (i)}{log_{2}(i+1)}
+
+
+\\
+DCG = {\color{Green}\frac{3}{log_{2}(2)} } + \frac{1}{log_{2}(3)} + {\color{Red}\frac{0}{log_{2}(4)} }+\frac{2}{log_{2}(5)} = 4.49
+
+\\
+
+(Ideal)\space DCG = {\color{Green}\frac{3}{log_{2}(2)} } + \frac{2}{log_{2}(3)}  + \frac{1}{log_{2}(4)} + {\color{Red}\frac{0}{log_{2}(5)} } = 5.88
+
+\\
+Normalized\space Discounted\space Cumulative\space Gain
+ = \frac{ DCG}{Ideal\space DCG} = \frac{4.49}{5.88}
+```

docs/internal_notes.md

@@ -0,0 +1,34 @@
+#name: workshop
+channels:
+  - conda-forge
+dependencies:
+  - python==3.7.*
+  - pip
+  - mamba
+  - nb_conda_kernels
+  - pyarrow==9.0.* 
+  - lxml==4.9.*
+  - pip:
+      - google-cloud-bigquery-storage 
+      - elasticsearch==8.4.*
+      - weaviate-client==3.8.0
+      - pandarallel==1.6.*
+      - milvus==2.1.*
+      - rich==12.6.*
+      - jupyterlab-execute-time==2.1.*
+      - sentence-transformers==2.2.*
+      - ipywidgets==8.0.*
+      - pyarrow==10.0.*
+      - gcsfs==2022.10.*
+      - papermill==2.3.*
+      - rank_bm25==0.2.*
+      - faiss_cpu==1.7.*
+
+
+
+# 
+#- google-cloud-bigquery-storage==2.16.* 
+      #- modin[ray]
+      # - pymilvus==1.1.0
+      #- pandas-gbq==0.17.*
+      #- git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[faiss]