While Enabling Elastic Search Getting Error

onkar_chavan · March 19, 2023, 11:26am

Hi Created my Own image and Then tried doing docker-compose up Getting following Error

tihomir · March 19, 2023, 5:18pm

Can you share your static config? (could bash into your server image and share /etc/temporal/config/docker.yaml)

ONKAR_CHAVAN1 · March 19, 2023, 5:55pm

I Copied the /etc/temporal/config/docker.yaml below:-

log:
stdout: true
level: info

persistence:
numHistoryShards: 4
defaultStore: default
visibilityStore: visibility
datastores:
default:
cassandra:
hosts: “”
keyspace: “temporal”
user: “”
password: “”
port: 9042
maxConns: 20
tls:
enabled: false
caFile:
certFile:
keyFile:
caData:
certData:
keyData:
enableHostVerification: false
serverName:
visibility:
cassandra:

            hosts: ""
            keyspace: "temporal_visibility"
            user: ""
            password: ""
            port: 9042
            maxConns: 10
            tls:
                enabled: false
                caFile: 
                certFile: 
                keyFile: 
                caData: 
                certData: 
                keyData: 
                enableHostVerification: false
                serverName:

global:
membership:
maxJoinDuration: 30s
broadcastAddress: “”
pprof:
port: 0
tls:
refreshInterval: 0s
expirationChecks:
warningWindow: 0s
errorWindow: 0s
checkInterval: 0s
internode:
# This server section configures the TLS certificate that internal temporal
# cluster nodes (history, matching, and internal-frontend) present to other
# clients within the Temporal Cluster.
server:
requireClientAuth: false

            certFile: 
            keyFile: 

            certData: 
            keyData: 

        # This client section is used to configure the TLS clients within
        # the Temporal Cluster that connect to an Internode (history, matching, or
        # internal-frontend)
        client:
            serverName: 
            disableHostVerification: false
    frontend:
        # This server section configures the TLS certificate that the Frontend
        # server presents to external clients.
        server:
            requireClientAuth: false
            certFile: 
            keyFile: 

            certData: 
            keyData: 

        # This client section is used to configure the TLS clients within
        # the Temporal Cluster (specifically the Worker role) that connect to the Frontend service
        client:
            serverName: 
            disableHostVerification: false
authorization:
    jwtKeyProvider:
        keySourceURIs:
        refreshInterval: 1m
    permissionsClaimName: permissions
    authorizer: 
    claimMapper:

services:
frontend:
rpc:
grpcPort: 7233
membershipPort: 6933
bindOnIP: 172.17.0.2

matching:
    rpc:
        grpcPort: 7235
        membershipPort: 6935
        bindOnIP: 172.17.0.2

history:
    rpc:
        grpcPort: 7234
        membershipPort: 6934
        bindOnIP: 172.17.0.2

worker:
    rpc:
        grpcPort: 7239
        membershipPort: 6939
        bindOnIP: 172.17.0.2

clusterMetadata:
enableGlobalNamespace: false
failoverVersionIncrement: 10
masterClusterName: “active”
currentClusterName: “active”
clusterInformation:
active:
enabled: true
initialFailoverVersion: 1
rpcName: “frontend”
rpcAddress: 127.0.0.1:7233

dcRedirectionPolicy:
policy: “noop”
toDC: “”

archival:
history:
state: “enabled”
enableRead: true
provider:
filestore:
fileMode: “0666”
dirMode: “0766”
visibility:
state: “enabled”
enableRead: true
provider:
filestore:
fileMode: “0666”
dirMode: “0766”

namespaceDefaults:
archival:
history:
state: “disabled”
URI: “file:///tmp/temporal_archival/development”
visibility:
state: “disabled”
URI: “file:///tmp/temporal_vis_archival/development”

dynamicConfigClient:
filepath: “/etc/temporal/config/dynamicconfig/docker.yaml”
pollInterval: “60s”

tihomir · March 20, 2023, 4:47am

Seems you dont have advanced visibility configured (you would see advancedVisibilityStore config in your static config. With this setup you are using standard visibility.

Can you share your docker compose?

ONKAR_CHAVAN1 · March 20, 2023, 5:18am

Hi @tihomir ,
Here is my docker-compose.yaml :-
version: “3.5”
services:
elasticsearch:
container_name: temporal-elasticsearch
environment:
- cluster.routing.allocation.disk.threshold_enabled=true
- cluster.routing.allocation.disk.watermark.low=512mb
- cluster.routing.allocation.disk.watermark.high=256mb
- cluster.routing.allocation.disk.watermark.flood_stage=128mb
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms256m -Xmx256m
- xpack.security.enabled=false
image: elasticsearch:${ELASTICSEARCH_VERSION}
networks:
- temporal-network
expose:
- 9200
volumes:
- /var/lib/elasticsearch/data
postgresql:
container_name: temporal-postgresql
environment:
POSTGRES_PASSWORD: temporal
POSTGRES_USER: temporal
image: postgres:${POSTGRESQL_VERSION}
networks:
- temporal-network
expose:
- 5432
volumes:
- /var/lib/postgresql/data
temporal:
container_name: temporal
depends_on:
- postgresql
- elasticsearch
environment:
- DB=postgresql
- DB_PORT=5432
- POSTGRES_USER=temporal
- POSTGRES_PWD=temporal
- POSTGRES_SEEDS=postgresql
- DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
- ENABLE_ES=true
- ES_SEEDS=elasticsearch
- ES_VERSION=v7
image: temporalio/auto-setup:onkar
networks:
- temporal-network
ports:
- 7233:7233
labels:
kompose.volume.type: configMap
volumes:
- ./dynamicconfig:/etc/temporal/config/dynamicconfig
temporal-admin-tools:
container_name: temporal-admin-tools
depends_on:
- temporal
environment:
- TEMPORAL_CLI_ADDRESS=temporal:7233
image: temporalio/admin-tools:onkar
networks:
- temporal-network
stdin_open: true
tty: true
temporal-ui:
container_name: temporal-ui
depends_on:
- temporal
environment:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CORS_ORIGINS=http://localhost:3000
image: temporalio/ui:${TEMPORAL_UI_VERSION}
networks:
- temporal-network
ports:
- 8080:8080
networks:
temporal-network:
driver: bridge
name: temporal-network

As you said I don’t have advanced visibility configured so can you guide me how to do that the
The setup I followed were:-
I cloned temporal docker-build doing that I found out that the folder cli, tctl, temporal were empty so then I Clone cli folder , tctl folder and temporal folder using git clone . After that I tried Building Images using following command step by step:-

cd docker/base-images
docker build . -f base-builder.Dockerfile -t temporalio/base-builder:onkar
docker build . -f base-server.Dockerfile -t temporalio/base-server:onkar
cd …/…
5 .docker build . -f server.Dockerfile -t temporalio/server:onkar
docker build . -f admin-tools.Dockerfile -t temporalio/admin-tools:onkar --build-arg SERVER_IMAGE=temporalio/server:onkar
docker build . -f auto-setup.Dockerfile -t temporalio/auto-setup:onkar2–build-arg SERVER_IMAGE=temporalio/server:onkar --build-arg ADMIN_TOOLS_IMAGE=temporalio/admin-tools:onkar

After Doing that I replaced my Image in docker-compose file which I clone using git clone GitHub - temporalio/docker-compose: Temporal docker-compose files
So as you said While build server image which is the 3 step I have done before that I have to enable advance visibility and then build the image so where do I have do those configurations(in which all files/folder specific)?

ONKAR_CHAVAN1 · March 21, 2023, 11:48am

Hi @tihomir, I have rebuilt the image using temporal/docker-build
Now I am able to see the advance visibility in below file.

Here is my /etc/temporal/config/docker.yaml:-
log:
stdout: true
level: info

persistence:
numHistoryShards: 4
defaultStore: default
visibilityStore: visibility
advancedVisibilityStore: es-visibility
datastores:
default:
sql:
pluginName: “postgres”
databaseName: “temporal”
connectAddr: “postgresql:5432”
connectProtocol: “tcp”
user: “temporal”
password: “temporal”
maxConns: 20
maxIdleConns: 20
maxConnLifetime: 1h
authPlugin:
plugin:
tls:
enabled: false
caFile:
certFile:
keyFile:
enableHostVerification: false
serverName:
visibility:
sql:

            pluginName: "postgres"
            databaseName: "temporal_visibility"
            connectAddr: "postgresql:5432"
            connectProtocol: "tcp"
            user: "temporal"
            password: "temporal"
            maxConns: 10
            maxIdleConns: 10
            maxConnLifetime: 1h
            authPlugin: 
                plugin: 
            tls:
                enabled: false
                caFile: 
                certFile: 
                keyFile: 
                enableHostVerification: false
                serverName: 
    es-visibility:
        elasticsearch:
            version: v7
            url:
                scheme: http
                host: "elasticsearch:9200"
            username: ""
            password: ""
            indices:
                visibility: "temporal_visibility_v1_dev"

global:
membership:
maxJoinDuration: 30s
broadcastAddress: “”
pprof:
port: 0
tls:
refreshInterval: 0s
expirationChecks:
warningWindow: 0s
errorWindow: 0s
checkInterval: 0s
internode:
# This server section configures the TLS certificate that internal temporal
# cluster nodes (history or matching) present to other clients within the Temporal Cluster.
server:
requireClientAuth: false

            certFile: 
            keyFile: 

            certData: 
            keyData: 

        # This client section is used to configure the TLS clients within
        # the Temporal Cluster that connect to an Internode (history or matching)
        client:
            serverName: 
            disableHostVerification: false
    frontend:
        # This server section configures the TLS certificate that the Frontend
        # server presents to all clients (specifically the Worker role within
        # the Temporal Cluster and all External SDKs connecting to the Cluster)
        server:
            requireClientAuth: false
            certFile: 
            keyFile: 

            certData: 
            keyData: 

        # This client section is used to configure the TLS clients within
        # the Temporal Cluster (specifically the Worker role) that connect to the Frontend service
        client:
            serverName: 
            disableHostVerification: false
authorization:
    jwtKeyProvider:
        keySourceURIs:
        refreshInterval: 1m
    permissionsClaimName: permissions
    authorizer: 
    claimMapper:

services:
frontend:
rpc:
grpcPort: 7233
membershipPort: 6933
bindOnIP: 172.21.0.2

matching:
    rpc:
        grpcPort: 7235
        membershipPort: 6935
        bindOnIP: 172.21.0.2

history:
    rpc:
        grpcPort: 7234
        membershipPort: 6934
        bindOnIP: 172.21.0.2

worker:
    rpc:
        grpcPort: 7239
        membershipPort: 6939
        bindOnIP: 172.21.0.2

clusterMetadata:
enableGlobalNamespace: false
failoverVersionIncrement: 10
masterClusterName: “active”
currentClusterName: “active”
clusterInformation:
active:
enabled: true
initialFailoverVersion: 1
rpcName: “frontend”
rpcAddress: 127.0.0.1:7233

dcRedirectionPolicy:
policy: “noop”
toDC: “”

archival:
history:
state: “enabled”
enableRead: true
provider:
filestore:
fileMode: “0666”
dirMode: “0766”
visibility:
state: “enabled”
enableRead: true
provider:
filestore:
fileMode: “0666”
dirMode: “0766”

namespaceDefaults:
archival:
history:
state: “disabled”
URI: “file:///tmp/temporal_archival/development”
visibility:
state: “disabled”
URI: “file:///tmp/temporal_vis_archival/development”

publicClient:
hostPort: “172.21.0.2:7233”

dynamicConfigClient:
filepath: “config/dynamicconfig/development-sql.yaml”
pollInterval: “60s”

Here is my docker-compose.yaml file:-
version: “3.5”
services:
elasticsearch:
container_name: temporal-elasticsearch
environment:
- cluster.routing.allocation.disk.threshold_enabled=true
- cluster.routing.allocation.disk.watermark.low=512mb
- cluster.routing.allocation.disk.watermark.high=256mb
- cluster.routing.allocation.disk.watermark.flood_stage=128mb
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms256m -Xmx256m
- xpack.security.enabled=false
image: elasticsearch:${ELASTICSEARCH_VERSION}
networks:
- temporal-network
expose:
- 9200
volumes:
- /var/lib/elasticsearch/data
postgresql:
container_name: temporal-postgresql
environment:
POSTGRES_PASSWORD: temporal
POSTGRES_USER: temporal
image: postgres:${POSTGRESQL_VERSION}
networks:
- temporal-network
expose:
- 5432
volumes:
- /var/lib/postgresql/data
temporal:
container_name: temporal
depends_on:
- postgresql
- elasticsearch
environment:
- DB=postgresql
- DB_PORT=5432
- POSTGRES_USER=temporal
- POSTGRES_PWD=temporal
- POSTGRES_SEEDS=postgresql
- DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
- ENABLE_ES=true
- ES_SEEDS=elasticsearch
- ES_VERSION=v7
image: temporalio/auto-setup:onkar20
networks:
- temporal-network
ports:
- 7233:7233
labels:
kompose.volume.type: configMap
volumes:
- ./dynamicconfig:/etc/temporal/config/dynamicconfig
temporal-admin-tools:
container_name: temporal-admin-tools
depends_on:
- temporal
environment:
- TEMPORAL_CLI_ADDRESS=temporal:7233
image: temporalio/admin-tools:onkar20
networks:
- temporal-network
stdin_open: true
tty: true
temporal-ui:
container_name: temporal-ui
depends_on:
- temporal
environment:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CORS_ORIGINS=http://localhost:3000
image: temporalio/ui:${TEMPORAL_UI_VERSION}
networks:
- temporal-network
ports:
- 8080:8080
networks:
temporal-network:
driver: bridge
name: temporal-network

Getting “no such index [temporal_visibility_v1_dev] and [action.auto_create_index] is [false]” Error

Any Help on this @tihomir