OpenSearchにローカルカスタムモデル(例: multilingual-e5-large-instruct)を導入するスクリプト。モデルデプロイの完了を待たない造りなので注意。最後にFess用の設定も出力する

setup_custom_local_model_for_opensearch.sh

opensearch_host=$1
pipeline_name=neural_pipeline
tmp_file=/tmp/output.$$

# AT FIRST, READ https://qiita.com/myu65/items/97a66dc8d06be9d99c74
# THEN, EDIT FOLLOWING VARIABLES FOR YOUR MODEL
model_name=intfloat/multilingual-e5-large-instruct
dimension=1024
space_type=cosinesimil
framework_type=huggingface_transformers
all_config="{\"_name_or_path\": \"intfloat/multilingual-e5-large-instruct\", \"architectures\": [\"XLMRobertaModel\"], \"attention_probs_dropout_prob\": 0.1,\"bos_token_id\": 0, \"classifier_dropout\": null, \"eos_token_id\": 2, \"export_model_type\": \"transformer\",\"hidden_act\": \"gelu\",\"hidden_dropout_prob\": 0.1,\"hidden_size\": 1024,\"initializer_range\": 0.02, \"intermediate_size\": 4096,\"layer_norm_eps\": 1e-05, \"max_position_embeddings\": 514,\"model_type\": \"xlm-roberta\", \"num_attention_heads\": 16,\"num_hidden_layers\": 24, \"output_past\": true,\"pad_token_id\": 1,\"position_embedding_type\": \"absolute\", \"torch_dtype\": \"float16\", \"transformers_version\": \"4.39.3\", \"type_vocab_size\": 1, \"use_cache\": true, \"vocab_size\": 250002 }"
description="This is a multilingual-e5-large-instruct model: It maps sentences & paragraphs to a 1024 dimensional dense vector space and can be used for tasks like clustering or semantic search."
model_format=ONNX
model_content_size_in_bytes=1257413818
model_content_hash_value="78c404fde39c3f98e14652fc5cf457a7e57c917891f40e374ba9b0b55f61409c"
url="http://fastapi_for_model/get_file/model/multilingual-e5-large-instruct.zip"
# FOR MORE DETAIL, READ https://docs.opensearch.org/docs/latest/ml-commons-plugin/api/model-apis/register-model

echo "Registering cluster settings..."
curl -o ${tmp_file} -s -XPUT -H "Content-Type:application/json" "${opensearch_host}/_cluster/settings" \
--data-raw '{
  "persistent": {
    "plugins.ml_commons.allow_registering_model_via_url": "true",
    "plugins.ml_commons.only_run_on_ml_node": "false",
    "plugins.ml_commons.model_access_control_enabled": "true",
    "plugins.ml_commons.native_memory_threshold": "100",
	"plugins.ml_commons.jvm_heap_memory_threshold": "95",
	"plugins.ml_commons.ml_task_timeout_in_seconds": "86400"
  }
}'

echo "Registering a language model..."
curl -o ${tmp_file} -s -XPOST -H "Content-Type:application/json" "${opensearch_host}/_plugins/_ml/model_groups/_register" \
--data-raw '{
  "name": "'${intfloat/multilingual-e5-large-instruct}'",
  "description": "A language model for semantic search."
}'

model_group_id=$(cat ${tmp_file} | ./jq -r .model_group_id)
echo "model group id: '${model_group_id}'"
if [[ ${model_group_id} = "null" ]] ; then
  echo "Failed to run a task: "$(cat ${tmp_file})
  rm -f $tmp_file
  exit 1
fi

echo "Uploading language model..."
curl -o ${tmp_file} -s -XPOST -H "Content-Type:application/json" "${opensearch_host}/_plugins/_ml/models/_register" \
--data-raw '{
	"name": "'${model_name}'",
	"version": "1.0.0",
    "model_group_id":"'${model_group_id}'",
	"description": '${description}',
	"model_task_type": "TEXT_EMBEDDING",
	"model_format": "'${model_format}'",
	"model_content_size_in_bytes": '${model_content_size_in_bytes}',
	"model_content_hash_value": '${model_content_hash_value}',
	"model_config": {
		"pooling_mode": "mean",
		"normalize_result": "true",
		"model_type": "xlm-roberta",
		"embedding_dimension": '${dimension}',
		"framework_type": "'${framework_type}'",
		"all_config": '${all_config}'
	},
	"created_time": 1676072210947,
    "url":'${url}'
}'

task_id=$(cat ${tmp_file} | jq -r .task_id)
if [[ ${task_id} = "null" ]] ; then
  echo "Failed to run a task: "$(cat ${tmp_file})
  rm -f $tmp_file
  exit 1
fi

echo -n "Checking task:${task_id}"
ret=RUNNING
while [ $ret = "CREATED" ] || [ $ret = "RUNNING" ] ; do
  sleep 1
  curl -o ${tmp_file} -s -XGET -H "Content-Type:application/json" "${opensearch_host}/_plugins/_ml/tasks/${task_id}"
  ret=$(cat ${tmp_file} | jq -r .state)
  model_id=$(cat ${tmp_file} | jq -r .model_id)
  echo -n "."
done
echo

curl -o ${tmp_file} -s -XPOST -H "Content-Type:application/json" "${opensearch_host}/_plugins/_ml/controllers/${model_id}" \
 --data-raw '{
  "user_rate_limiter": {
    "user1": {
      "limit": 4,
      "unit": "MINUTES"
    }
  }
}'
echo $(cat ${tmp_file})

curl -o ${tmp_file} -s -XPOST -H "Content-Type:application/json" ${opensearch_host}/_plugins/_ml/models/${model_id}/_deploy
echo $(cat ${tmp_file}) 

echo "Setting pipeline:${pipeline_name}..."
curl -o ${tmp_file} -s -XPUT -H "Content-Type:application/json" "${opensearch_host}/_ingest/pipeline/${pipeline_name}" \
--data-raw '{
  "description": "A neural search pipeline",
  "processors" : [
    {
      "text_chunking": {
        "algorithm": {
          "fixed_token_length": {
            "token_limit": 300,
            "overlap_rate": 0.1,
            "tokenizer": "standard"
          }
        },
        "field_map": {
          "content": "content_chunk"
        }
      }
    },
    {
      "text_embedding": {
        "model_id": "'"${model_id}"'",
        "field_map": {
           "content_chunk": "content_vector"
        }
      }
    }
  ]
}'

acknowledged=$(cat ${tmp_file} | ./jq -r .acknowledged)
if [[ ${acknowledged} != "true" ]] ; then
  echo "Failed to craete ${pipeline_name}"$(cat ${tmp_file})
  rm -f $tmp_file
  exit 1
fi


# IF YOU USE OPENSEARCH FOR FESS, COPY AND PASTE FOLLOWING VALUES AS SYSTEM PROPERTIES OF FESS.
cat << EOS
--- system properties: start ---
fess.semantic_search.pipeline=${pipeline_name}
fess.semantic_search.content.nested_field=content_vector
fess.semantic_search.content.chunk_field=content_chunk
fess.semantic_search.content.field=knn
fess.semantic_search.content.dimension=${dimension}
fess.semantic_search.content.method=hnsw
fess.semantic_search.content.engine=lucene
fess.semantic_search.content.space_type=${space_type}
fess.semantic_search.content.model_id=${model_id}
fess.semantic_search.min_score=0.5
--- system properties: end ---
EOS

rm -f $tmp_file

# https://zenyasai.g-fukurowl.club/@g_fukurowl_zenyasai/114366909734834941