-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathMakefile
125 lines (100 loc) · 3.32 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
# https://github.com/aaronland/go-tools
URLESCAPE=$(shell which urlescape)
# Opensearch server
# This is for debugging. Do not change this at your own risk.
# (That means you should change this.)
OS_PSWD=KJHFGDFJGSJfsdkjfhsdoifruwo45978h52dcn
OS_MODEL=9dgHD5ABSoo-6k3cWDqn
# If true this tends to trigger Java heap memory errors for OS run inside a Docker container
OS_BULK=false
OS_DSN="https://localhost:9200/dedupe?username=admin&password=$(OS_PSWD)&insecure=true&require-tls=true"
ENC_OS_DSN=$(shell $(URLESCAPE) $(OS_DSN))
OS_DATABASE_URI=opensearch://?dsn=$(ENC_OS_DSN)&model=$(OS_MODEL)&bulk-index=$(OS_BULK)
# https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/
#
# And then:
# curl -v -k https://admin:$(OS_PSWD)@localhost:9200/
# -it \
local-server:
docker run \
-p 9200:9200 \
-p 9600:9600 \
-e "discovery.type=single-node" \
-e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(OS_PSWD)" \
-v opensearch-data1:/usr/local/data/opensearch \
opensearchproject/opensearch:latest
# Quick and dirty targets for testing things
local-aliases:
curl -k -s \
-H 'Content-Type: application/json' \
-X GET \
https://admin:$(OS_PSWD)@localhost:9200/_aliases \
| jq
local-settings:
curl -k -s \
-H 'Content-Type: application/json' \
-X GET \
https://admin:$(OS_PSWD)@localhost:9200/dedupe/_settings \
| jq
local-mappings:
curl -k -s \
-H 'Content-Type: application/json' \
-X GET \
https://admin:$(OS_PSWD)@localhost:9200/dedupe/_mappings \
| jq
local-search:
curl -k -s \
-H 'Content-Type: application/json' \
-X GET \
https://admin:$(OS_PSWD)@localhost:9200/dedupe/_search \
| json_pp
# https://opensearch.org/docs/latest/ml-commons-plugin/pretrained-models/
# https://opensearch.org/blog/improving-document-retrieval-with-sparse-semantic-encoders/
# step 1: local-config-cluster-settings
# step 3: local-config-register-model
# step 4: local-task-status TASK=<task-id> and record <model-id>
# step 5: write <model-id> to static/opensearch/dedupe-ingest-pipeline-*.json
# step 6: local-config-pipeline
# step 7: local-config-index
# step 8: local-index-overture DATA=<path>
local-config-cluster-settings:
cat static/opensearch/cluster-settings.json | \
curl -k -s \
-H 'Content-Type: application/json' \
-X PUT \
https://admin:$(OS_PSWD)@localhost:9200/_cluster/settings \
-d @-
local-config-register-model:
cat static/opensearch/register-model-text.json | \
curl -k -s \
-H 'Content-Type: application/json' \
-X POST \
'https://admin:$(OS_PSWD)@localhost:9200/_plugins/_ml/models/_register?deploy=true' \
-d @-
# To do: Extract model_id...
local-task-status:
curl -k -s \
-H 'Content-Type: application/json' \
-X GET \
https://admin:$(OS_PSWD)@localhost:9200/_plugins/_ml/tasks/$(TASK) \
| jq
# To do: Write / insert model_id..
local-config-pipeline:
cat static/opensearch/dedupe-ingest-pipeline-text.json | \
curl -k -s \
-H 'Content-Type: application/json' \
-X PUT \
https://admin:$(OS_PSWD)@localhost:9200/_ingest/pipeline/dedupe-ingest-pipeline \
-d @-
local-remove-index:
curl -k -s \
-H 'Content-type:application/json' \
-XDELETE \
https://admin:$(OS_PSWD)@localhost:9200/dedupe
local-config-index:
cat static/opensearch/dedupe-index-text.json | \
curl -k -s \
-H 'Content-type:application/json' \
-XPUT \
https://admin:$(OS_PSWD)@localhost:9200/dedupe \
-d @-