Module 2 (#5)

.github/workflows/module-2.yaml

@@ -0,0 +1,42 @@
+name: Module 2
+
+
+on:
+  workflow_dispatch:
+  # push:
+  pull_request:
+    branches:
+      - main
+
+
+jobs:
+
+  minio-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10' 
+
+      - name: Run minio
+        run: |
+          docker run -d -p 9000:9000 -p 9001:9001 quay.io/minio/minio server /data --console-address ":9001"
+
+        
+      - name: Setup env
+        run: |
+          pip install -r module-2/requirements.test.txt
+
+      - name: Run test
+        run: |
+          export AWS_ACCESS_KEY_ID=minioadmin
+          export AWS_SECRET_ACCESS_KEY=minioadmin
+          export AWS_ENDPOINT_URL=http://127.0.0.1:9000        
+          aws s3api create-bucket --bucket test
+          pytest -ss ./module-2/minio_storage/test_minio_client.py

module-2/dvc_test/.dvc/.gitignore → module-2/.dvc/.gitignore

@@ -1,3 +1,2 @@
 /config.local
 /tmp
-/cache

module-2/.dvc/config

@@ -0,0 +1,5 @@
+[core]
+    remote = minio
+['remote "minio"']
+    url = s3://ml-data
+    endpointurl = http://127.0.0.1:9000

module-2/.gitignore

@@ -0,0 +1,4 @@
+random-data
+.lancedb/
+.ruff_cache
+cache

module-2/PRACTICE.md

@@ -1,11 +1,10 @@
-# Practice 
+# Practice
 
-*** 
+***
 
 # H3: Data storage & processing
 
-## Reading list: 
-
+## Reading list:
 
 - [Data engineer roadmap](https://github.com/datastacktv/data-engineer-roadmap)
 - [Minio using Kubernetes](https://github.com/kubernetes/examples/tree/master/staging/storage/minio)
@@ -24,7 +23,6 @@
 - [Course: CMU Database Systems](https://15445.courses.cs.cmu.edu/fall2023/)
 - [Course: Advanced Database Systems](https://15721.courses.cs.cmu.edu/spring2024/)
 
-
 ## Task:
 
 - PR1: Write README instructions detailing how to deploy MinIO with the following options: Local, Docker, Kubernetes (K8S)-based.
@@ -35,10 +33,9 @@
 - PR7: Write code for transforming your dataset into a vector format, and utilize VectorDB for ingestion and querying.
 - Google Doc: Update your proposal by adding a section on data storage and processing.
 
+## Criteria:
 
-## Criteria: 
-
-- 7 PRs are merged 
+- 7 PRs are merged
 - Description of data section, storage and processing, in the google doc.