HDDS-12113. Move HAProxy test to HA environment (apache#8271)

Co-authored-by: Doroszlai, Attila <[email protected]>

Author: peterxcli

hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml

@@ -34,13 +34,7 @@ services:
     command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
   s3g1:
     <<: *s3-worker
-    ports:
-      - 9879:9878
   s3g2:
     <<: *s3-worker
-    ports:
-      - 9880:9878
   s3g3:
     <<: *s3-worker
-    ports:
-      - 9881:9878

…ompose/ozone/disabled-test-s3-haproxy.sh …ain/compose/ozone-ha/test-haproxy-s3g.shhadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh renamed to hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh renamed to hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh

@@ -15,16 +15,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#suite:HA-unsecure
+
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
 
 export SECURITY_ENABLED=false
 export COMPOSE_FILE=docker-compose.yaml:../common/s3-haproxy.yaml
+export OZONE_REPLICATION_FACTOR=3
+export SCM=scm1
 
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
 start_docker_env
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-execute_robot_test scm --exclude virtual-host s3
+exclude="--exclude virtual-host"
+for bucket in generated; do
+  execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  # some tests are independent of the bucket type, only need to be run once
+  ## Exclude awss3virtualhost.robot
+  exclude="--exclude virtual-host --exclude no-bucket-type"
+done

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml

@@ -14,6 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+x-common-config:
+  &common-config
+  dns_search: .
+  image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+  volumes:
+    - ../..:/opt/hadoop
+    - ../_keytabs:/etc/security/keytabs
+    - ./krb5.conf:/etc/krb5.conf
+  env_file:
+    - docker-config
+
 services:
   kdc:
     image: ${OZONE_TESTKRB5_IMAGE}
@@ -44,12 +55,7 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.101
   datanode1:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 19864:9999
     command: ["/opt/hadoop/bin/ozone","datanode"]
@@ -58,21 +64,14 @@ services:
       - "scm2.org: 172.25.0.117"
       - "scm3.org: 172.25.0.118"
       - "recon: 172.25.0.115"
-    env_file:
-      - docker-config
     environment:
       WAITFOR: scm3.org:9894
       OZONE_OPTS:
     networks:
       ozone_net:
         ipv4_address: 172.25.0.102
   datanode2:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9866:9999
     command: ["/opt/hadoop/bin/ozone","datanode"]
@@ -81,21 +80,14 @@ services:
       - "scm2.org: 172.25.0.117"
       - "scm3.org: 172.25.0.118"
       - "recon: 172.25.0.115"
-    env_file:
-      - docker-config
     environment:
       WAITFOR: scm3.org:9894
       OZONE_OPTS:
     networks:
       ozone_net:
         ipv4_address: 172.25.0.103
   datanode3:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9868:9999
     command: ["/opt/hadoop/bin/ozone","datanode"]
@@ -104,22 +96,15 @@ services:
       - "scm2.org: 172.25.0.117"
       - "scm3.org: 172.25.0.118"
       - "recon: 172.25.0.115"
-    env_file:
-      - docker-config
     environment:
       WAITFOR: scm3.org:9894
       OZONE_OPTS:
     networks:
       ozone_net:
         ipv4_address: 172.25.0.104
   om1:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: om1
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9880:9874
       - 9890:9872
@@ -128,8 +113,6 @@ services:
       WAITFOR: scm3.org:9894
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
       OZONE_OPTS:
-    env_file:
-      - ./docker-config
     command: ["/opt/hadoop/bin/ozone","om"]
     extra_hosts:
       - "scm1.org: 172.25.0.116"
@@ -139,13 +122,8 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.111
   om2:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: om2
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9882:9874
       - 9892:9872
@@ -154,8 +132,6 @@ services:
       WAITFOR: scm3.org:9894
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
       OZONE_OPTS:
-    env_file:
-      - ./docker-config
     command: ["/opt/hadoop/bin/ozone","om"]
     extra_hosts:
       - "scm1.org: 172.25.0.116"
@@ -165,13 +141,8 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.112
   om3:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: om3
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9884:9874
       - 9894:9872
@@ -180,8 +151,6 @@ services:
       WAITFOR: scm3.org:9894
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
       OZONE_OPTS:
-    env_file:
-      - ./docker-config
     command: ["/opt/hadoop/bin/ozone","om"]
     extra_hosts:
       - "scm1.org: 172.25.0.116"
@@ -191,17 +160,10 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.113
   httpfs:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: httpfs
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 14000:14000
-    env_file:
-      - ./docker-config
     command: [ "/opt/hadoop/bin/ozone","httpfs" ]
     environment:
       OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
@@ -210,31 +172,15 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.119
   s3g:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: s3g
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    ports:
-      - 9878:9878
-    env_file:
-      - ./docker-config
-    command: ["/opt/hadoop/bin/ozone","s3g"]
-    environment:
-      OZONE_OPTS:
+    <<: *common-config
+    command: ["ozone","s3g"]
     networks:
       ozone_net:
-        ipv4_address: 172.25.0.114
+        ipv4_address: 172.25.0.120
   scm1.org:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: scm1.org
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9990:9876
       - 9992:9860
@@ -256,18 +202,11 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.116
   scm2.org:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: scm2.org
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9994:9876
       - 9996:9860
-    env_file:
-      - docker-config
     environment:
       WAITFOR: scm1.org:9894
       ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION
@@ -285,18 +224,11 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.117
   scm3.org:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: scm3.org
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9998:9876
       - 10002:9860
-    env_file:
-      - docker-config
     environment:
       WAITFOR: scm2.org:9894
       ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION
@@ -314,17 +246,10 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.118
   recon:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     hostname: recon
-    dns_search: .
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *common-config
     ports:
       - 9888:9888
-    env_file:
-      - ./docker-config
     environment:
       OZONE_OPTS:
     command: ["/opt/hadoop/bin/ozone","recon"]

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Simple configuration for an HTTP proxy listening on port 9878 on all
+# interfaces and forwarding requests to a multiple multiple S3 servers in round
+# robin fashion.
+global
+    daemon
+    maxconn 256
+
+defaults
+    mode http
+    timeout connect 5000ms
+    timeout client 50000ms
+    timeout server 50000ms
+
+frontend http-in
+    bind *:9878
+    default_backend servers
+
+backend servers
+    balance roundrobin
+    server server1 172.25.0.121:9878 maxconn 32
+    server server2 172.25.0.122:9878 maxconn 32
+    server server3 172.25.0.123:9878 maxconn 32
+
+frontend webadmin
+    bind *:19878
+    default_backend webadmin-servers
+
+backend webadmin-servers
+    balance roundrobin
+    server server1 172.25.0.121:19878 maxconn 32
+    server server2 172.25.0.122:19878 maxconn 32
+    server server3 172.25.0.123:19878 maxconn 32