apache-dolphinscheduler-3.2.0 单机启动及改为mysql存储

原创已于 2023-11-17 08:14:55 修改 · 1.5k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#apache #mysql #数据库

于 2023-11-17 08:09:24 首次发布

大数据同时被 3 个专栏收录

57 篇文章

订阅专栏

JAVA知识

33 篇文章

订阅专栏

机器学习

13 篇文章

订阅专栏

一、apache-dolphinscheduler介绍

Apache DolphinScheduler 是一个分布式的开源调度系统，它提供了一种可视化、可扩展、高可用的任务调度和数据处理方案。

二、apache-dolphinscheduler配置

2.1、下载apache-dolphinscheduler

https://dolphinscheduler.apache.org/en-us/download/3.2.0

apache-dolphinscheduler-3.2.0

2.2、H2配置文件

spring:
  jackson:
    time-zone: UTC
    date-format: "yyyy-MM-dd HH:mm:ss"
  banner:
    charset: UTF-8
  cache:
    # default enable cache, you can disable by `type: none`
    type: none
    cache-names:
      - tenant
      - user
      - processDefinition
      - processTaskRelation
      - taskDefinition
    caffeine:
      spec: maximumSize=100,expireAfterWrite=300s,recordStats
  sql:
    init:
      schema-locations: classpath:sql/dolphinscheduler_h2.sql
  datasource:
    driver-class-name: org.h2.Driver
    url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
    username: sa
    password: ""


  quartz:
    job-store-type: jdbc
    jdbc:
      initialize-schema: never
    properties:
      org.quartz.threadPool.threadPriority: 5
      org.quartz.jobStore.isClustered: true
      org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
      org.quartz.scheduler.instanceId: AUTO
      org.quartz.jobStore.tablePrefix: QRTZ_
      org.quartz.jobStore.acquireTriggersWithinLock: true
      org.quartz.scheduler.instanceName: DolphinScheduler
      org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
      org.quartz.jobStore.useProperties: false
      org.quartz.threadPool.makeThreadsDaemons: true
      org.quartz.threadPool.threadCount: 25
      org.quartz.jobStore.misfireThreshold: 60000
      org.quartz.scheduler.makeSchedulerThreadDaemon: true
      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
      org.quartz.jobStore.clusterCheckinInterval: 5000
      org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
  servlet:
    multipart:
      max-file-size: 1024MB
      max-request-size: 1024MB
  messages:
    basename: i18n/messages
  jpa:
    hibernate:
      ddl-auto: none
  mvc:
    pathmatch:
      matching-strategy: ANT_PATH_MATCHER

registry:
  type: zookeeper
  zookeeper:
    namespace: dolphinscheduler
    connect-string: localhost:2181
    retry-policy:
      base-sleep-time: 60ms
      max-sleep: 300ms
      max-retries: 5
    session-timeout: 30s
    connection-timeout: 9s
    block-until-connected: 600ms
    digest: ~

security:
  authentication:
    # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
    type: PASSWORD
    # IF you set type `LDAP`, below config will be effective
    ldap:
      # ldap server config
      urls: ldap://ldap.forumsys.com:389/
      base-dn: dc=example,dc=com
      username: cn=read-only-admin,dc=example,dc=com
      password: password
      user:
        # admin userId when you use LDAP login
        admin: read-only-admin
        identity-attribute: uid
        email-attribute: mail
        # action when ldap user is not exist (supported types: CREATE,DENY)
        not-exist-action: CREATE
      ssl:
        enable: false
        # jks file absolute path && password
        trust-store: "/ldapkeystore.jks"
        trust-store-password: ""
    casdoor:
      user:
        admin: admin
    oauth2:
      enable: false
      provider:
        github:
          authorizationUri: "https://github.com/login/oauth/authorize"
          redirectUri: "http://localhost:12345/dolphinscheduler/redirect/login/oauth2"
          clientId: ""
          clientSecret: ""
          tokenUri: "https://github.com/login/oauth/access_token"
          userInfoUri: "https://api.github.com/user"
          callbackUrl: "http://localhost:5173/login"
          iconUri: ""
          provider: github
        gitee:
          authorizationUri: "https://gitee.com/oauth/authorize"
          redirectUri: "http://127.0.0.1:12345/dolphinscheduler/redirect/login/oauth2"
          clientId: ""
          clientSecret: ""
          tokenUri: "https://gitee.com/oauth/token?grant_type=authorization_code"
          userInfoUri: "https://gitee.com/api/v5/user"
          callbackUrl: "http://127.0.0.1:5173/login"
          iconUri: ""
          provider: gitee

casdoor:
  # Your Casdoor server url
  endpoint: http://localhost:8000
  client-id: ""
  client-secret: ""
  # The certificate may be multi-line, you can use `|-` for ease
  certificate: ""
  # Your organization name added in Casdoor
  organization-name: built-in
  # Your application name added in Casdoor
  application-name: dolphinscheduler
  # Doplhinscheduler login url
  redirect-url: http://localhost:5173/login



master:
  listen-port: 5678
  # master fetch command num
  fetch-command-num: 10
  # master prepare execute thread number to limit handle commands in parallel
  pre-exec-threads: 10
  # master execute thread number to limit process instances in parallel
  exec-threads: 10
  # master dispatch task number per batch
  dispatch-task-number: 3
  # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
  host-selector: lower_weight
  # master heartbeat interval
  heartbeat-interval: 10s
  # master commit task retry times
  task-commit-retry-times: 5
  # master commit task interval
  task-commit-interval: 1s
  state-wheel-interval: 5s
  # master max cpuload avg percentage, only higher than the system cpu load average, master server can schedule. default value 1: will use 100% cpu
  max-cpu-load-avg: 1
  # master reserved memory, only lower than system available memory, master server can schedule. default value 0.1, only the available memory is higher than 10%, master server can schedule.
  reserved-memory: 0.1
  # failover interval
  failover-interval: 10m
  # kill yarn/k8s application when failover taskInstance, default true
  kill-application-when-task-failover: true
  worker-group-refresh-interval: 10s

worker:
  # worker listener port
  listen-port: 1234
  # worker execute thread number to limit task instances in parallel
  exec-threads: 10
  # worker heartbeat interval
  heartbeat-interval: 10s
  # worker host weight to dispatch tasks, default value 100
  host-weight: 100
  # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
  tenant-auto-create: true
  #Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
  tenant-distributed-user: false
  # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value 1: will use 100% cpu.
  max-cpu-load-avg: 1
  # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.1, only the available memory is higher than 10%, worker server can receive task.
  reserved-memory: 0.1
  task-execute-threads-full-policy: REJECT

alert:
  port: 50052
  # Mark each alert of alert server if late after x milliseconds as failed.
  # Define value is (0 = infinite), and alert server would be waiting alert result.
  wait-timeout: 0
  heartbeat-interval: 60s

api:
  audit-enable: false
  # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
  # global max request number per second
  # default tenant-level max request number
  traffic-control:
    global-switch: false
    max-global-qps-rate: 300
    tenant-switch: false
    default-tenant-qps-rate: 10
      #customize-tenant-qps-rate:
    # eg.
    #tenant1: 11
    #tenant2: 20
  python-gateway:
    # Weather enable python gateway server or not. The default value is true.
    enabled: true
    # Authentication token for connection from python api to python gateway server. Should be changed the default value
    # when you deploy in public network.
    auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
    # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
    # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
    gateway-server-address: 0.0.0.0
    # The port of Python gateway server start. Define which port you could connect to Python gateway server from
    # Python API side.
    gateway-server-port: 25333
    # The address of Python callback client.
    python-address: 127.0.0.1
    # The port of Python callback client.
    python-port: 25334
    # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
    # and socket server would never close even though no requests accept
    connect-timeout: 0
    # Close each active connection of socket server if python program not active after x milliseconds. Define value is
    # (0 = infinite), and socket server would never close even though no requests accept
    read-timeout: 0

server:
  port: 12345
  servlet:
    session:
      timeout: 120m
    context-path: /dolphinscheduler/
  compression:
    enabled: true
    mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
  jetty:
    max-http-form-post-size: 5000000

management:
  endpoints:
    web:
      exposure:
        include: health,metrics,prometheus
  endpoint:
    health:
      enabled: true
      show-details: always
  health:
    db:
      enabled: true
    defaults:
      enabled: false
  metrics:
    tags:
      application: ${spring.application.name}

metrics:
  enabled: true

# Override by profile
---
spring:
  config:
    activate:
      on-profile: postgresql
  quartz:
    properties:
      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
  datasource:
    driver-class-name: org.postgresql.Driver
    url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
    username: root
    password: root

---
spring:
  config:
    activate:
      on-profile: mysql
  sql:
     init:
       schema-locations: classpath:sql/dolphinscheduler_mysql.sql
  datasource:
    driver-class-name: com.mysql.cj.jdbc.Driver
    url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
    username: root
    password: root

2.2、mysql的配置

spring:
  jackson:
    time-zone: UTC
    date-format: "yyyy-MM-dd HH:mm:ss"
  banner:
    charset: UTF-8
  cache:
    # default enable cache, you can disable by `type: none`
    type: none
    cache-names:
      - tenant
      - user
      - processDefinition
      - processTaskRelation
      - taskDefinition
    caffeine:
      spec: maximumSize=100,expireAfterWrite=300s,recordStats
  sql:
    init:
      schema-locations: classpath:sql/dolphinscheduler_h2.sql
  datasource:
    #driver-class-name: org.h2.Driver
    #url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
    #username: sa
    #password: ""
    driver-class-name: com.mysql.cj.jdbc.Driver
    url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
    username: root
    password: 123456


  quartz:
    job-store-type: jdbc
    jdbc:
      initialize-schema: never
    properties:
      org.quartz.threadPool.threadPriority: 5
      org.quartz.jobStore.isClustered: true
      org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
      org.quartz.scheduler.instanceId: AUTO
      org.quartz.jobStore.tablePrefix: QRTZ_
      org.quartz.jobStore.acquireTriggersWithinLock: true
      org.quartz.scheduler.instanceName: DolphinScheduler
      org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
      org.quartz.jobStore.useProperties: false
      org.quartz.threadPool.makeThreadsDaemons: true
      org.quartz.threadPool.threadCount: 25
      org.quartz.jobStore.misfireThreshold: 60000
      org.quartz.scheduler.makeSchedulerThreadDaemon: true
      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
      org.quartz.jobStore.clusterCheckinInterval: 5000
      org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
  servlet:
    multipart:
      max-file-size: 1024MB
      max-request-size: 1024MB
  messages:
    basename: i18n/messages
  jpa:
    hibernate:
      ddl-auto: none
  mvc:
    pathmatch:
      matching-strategy: ANT_PATH_MATCHER

registry:
  type: zookeeper
  zookeeper:
    namespace: dolphinscheduler
    connect-string: localhost:2181
    retry-policy:
      base-sleep-time: 60ms
      max-sleep: 300ms
      max-retries: 5
    session-timeout: 30s
    connection-timeout: 9s
    block-until-connected: 600ms
    digest: ~

security:
  authentication:
    # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
    type: PASSWORD
    # IF you set type `LDAP`, below config will be effective
    ldap:
      # ldap server config
      urls: ldap://ldap.forumsys.com:389/
      base-dn: dc=example,dc=com
      username: cn=read-only-admin,dc=example,dc=com
      password: password
      user:
        # admin userId when you use LDAP login
        admin: read-only-admin
        identity-attribute: uid
        email-attribute: mail
        # action when ldap user is not exist (supported types: CREATE,DENY)
        not-exist-action: CREATE
      ssl:
        enable: false
        # jks file absolute path && password
        trust-store: "/ldapkeystore.jks"
        trust-store-password: ""
    casdoor:
      user:
        admin: admin
    oauth2:
      enable: false
      provider:
        github:
          authorizationUri: "https://github.com/login/oauth/authorize"
          redirectUri: "http://localhost:12345/dolphinscheduler/redirect/login/oauth2"
          clientId: ""
          clientSecret: ""
          tokenUri: "https://github.com/login/oauth/access_token"
          userInfoUri: "https://api.github.com/user"
          callbackUrl: "http://localhost:5173/login"
          iconUri: ""
          provider: github
        gitee:
          authorizationUri: "https://gitee.com/oauth/authorize"
          redirectUri: "http://127.0.0.1:12345/dolphinscheduler/redirect/login/oauth2"
          clientId: ""
          clientSecret: ""
          tokenUri: "https://gitee.com/oauth/token?grant_type=authorization_code"
          userInfoUri: "https://gitee.com/api/v5/user"
          callbackUrl: "http://127.0.0.1:5173/login"
          iconUri: ""
          provider: gitee

casdoor:
  # Your Casdoor server url
  endpoint: http://localhost:8000
  client-id: ""
  client-secret: ""
  # The certificate may be multi-line, you can use `|-` for ease
  certificate: ""
  # Your organization name added in Casdoor
  organization-name: built-in
  # Your application name added in Casdoor
  application-name: dolphinscheduler
  # Doplhinscheduler login url
  redirect-url: http://localhost:5173/login



master:
  listen-port: 5678
  # master fetch command num
  fetch-command-num: 10
  # master prepare execute thread number to limit handle commands in parallel
  pre-exec-threads: 10
  # master execute thread number to limit process instances in parallel
  exec-threads: 10
  # master dispatch task number per batch
  dispatch-task-number: 3
  # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
  host-selector: lower_weight
  # master heartbeat interval
  heartbeat-interval: 10s
  # master commit task retry times
  task-commit-retry-times: 5
  # master commit task interval
  task-commit-interval: 1s
  state-wheel-interval: 5s
  # master max cpuload avg percentage, only higher than the system cpu load average, master server can schedule. default value 1: will use 100% cpu
  max-cpu-load-avg: 1
  # master reserved memory, only lower than system available memory, master server can schedule. default value 0.1, only the available memory is higher than 10%, master server can schedule.
  reserved-memory: 0.1
  # failover interval
  failover-interval: 10m
  # kill yarn/k8s application when failover taskInstance, default true
  kill-application-when-task-failover: true
  worker-group-refresh-interval: 10s

worker:
  # worker listener port
  listen-port: 1234
  # worker execute thread number to limit task instances in parallel
  exec-threads: 10
  # worker heartbeat interval
  heartbeat-interval: 10s
  # worker host weight to dispatch tasks, default value 100
  host-weight: 100
  # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
  tenant-auto-create: true
  #Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
  tenant-distributed-user: false
  # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value 1: will use 100% cpu.
  max-cpu-load-avg: 1
  # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.1, only the available memory is higher than 10%, worker server can receive task.
  reserved-memory: 0.1
  task-execute-threads-full-policy: REJECT

alert:
  port: 50052
  # Mark each alert of alert server if late after x milliseconds as failed.
  # Define value is (0 = infinite), and alert server would be waiting alert result.
  wait-timeout: 0
  heartbeat-interval: 60s

api:
  audit-enable: false
  # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
  # global max request number per second
  # default tenant-level max request number
  traffic-control:
    global-switch: false
    max-global-qps-rate: 300
    tenant-switch: false
    default-tenant-qps-rate: 10
      #customize-tenant-qps-rate:
    # eg.
    #tenant1: 11
    #tenant2: 20
  python-gateway:
    # Weather enable python gateway server or not. The default value is true.
    enabled: true
    # Authentication token for connection from python api to python gateway server. Should be changed the default value
    # when you deploy in public network.
    auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
    # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
    # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
    gateway-server-address: 0.0.0.0
    # The port of Python gateway server start. Define which port you could connect to Python gateway server from
    # Python API side.
    gateway-server-port: 25333
    # The address of Python callback client.
    python-address: 127.0.0.1
    # The port of Python callback client.
    python-port: 25334
    # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
    # and socket server would never close even though no requests accept
    connect-timeout: 0
    # Close each active connection of socket server if python program not active after x milliseconds. Define value is
    # (0 = infinite), and socket server would never close even though no requests accept
    read-timeout: 0

server:
  port: 12345
  servlet:
    session:
      timeout: 120m
    context-path: /dolphinscheduler/
  compression:
    enabled: true
    mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
  jetty:
    max-http-form-post-size: 5000000

management:
  endpoints:
    web:
      exposure:
        include: health,metrics,prometheus
  endpoint:
    health:
      enabled: true
      show-details: always
  health:
    db:
      enabled: true
    defaults:
      enabled: false
  metrics:
    tags:
      application: ${spring.application.name}

metrics:
  enabled: true

# Override by profile
---
spring:
  config:
    activate:
      on-profile: postgresql
  quartz:
    properties:
      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
  datasource:
    driver-class-name: org.postgresql.Driver
    url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
    username: root
    password: root

---
spring:
  config:
    activate:
      on-profile: mysql
  sql:
     init:
       schema-locations: classpath:sql/dolphinscheduler_mysql.sql
  datasource:
    driver-class-name: com.mysql.cj.jdbc.Driver
    url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
    username: root
    password: root

2.3、dolphinscheduler_env.sh配置

export JAVA_HOME=${JAVA_HOME:-/usr/local/jdk1.8.0_391}
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-node4:2181}
export DATABASE=${DATABASE:-mysql}
export SPRING_PROFILES_ACTIVE=${DATABASE}
export SPRING_DATASOURCE_URL="jdbc:mysql://192.168.1.103:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true"
export SPRING_DATASOURCE_USERNAME=root
export SPRING_DATASOURCE_PASSWORD=123456

三、dolphinscheduler操作

3.1、启动dolphinscheduler

 ./dolphinscheduler-daemon.sh start standalone-server

默认是采用H2的内存数据库，所以任务不能创建太多。

3.2、查看dolphinscheduler状态

[root@node4 bin]# ./dolphinscheduler-daemon.sh status standalone-server
Begin status standalone-server......
standalone-server  [  RUNNING  ]
End status standalone-server.

3.3、停止dolphinscheduler

 ./dolphinscheduler-daemon.sh stop standalone-server

3.4、查看dolphinscheduler进程

[root@node4 bin]# jps
36069 Jps
35849 StandaloneServer

3.5、访问dolphinscheduler

http://192.168.42.142:12345/dolphinscheduler/ui

默认登录用户名/密码:admin/dolphinscheduler123

3.6、访问界面