Skip to content

[Feat]merge ps-cs, ps-data-source-manager and ps-metadataquery to ps-publicservice #2927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
peacewong merged 9 commits into from
Aug 25, 2022
Merged

[Feat]merge ps-cs, ps-data-source-manager and ps-metadataquery to ps-publicservice #2927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8676bb7
feat(public-enhancements): merge cs,datasource-manager, metadatequery…
jackxu2011 Aug 22, 2022
39602ec
fix: fix label register
jackxu2011 Aug 22, 2022
c61b485
fix: unify commons-math3 dependency, fix dependencies
jackxu2011 Aug 22, 2022
fc105b5
fix: remove the useless linkis-start-all.sh and linkis-stop-all.sh's …
jackxu2011 Aug 22, 2022
c8a4749
fix: fix the HaContextGatewayRouter for ps-cs service
jackxu2011 Aug 23, 2022
d75c76b
fix: fix the restful route for cs publicServiceList, and add enable_p…
jackxu2011 Aug 24, 2022
2f171c5
fix: fix duplicate declaration of some jars
jackxu2011 Aug 24, 2022
a586f1d
fix: comment the linkis.properties's publicservice.list conf, use the…
jackxu2011 Aug 24, 2022
8e40e3d
fix: add metedataquery contextPath to public_service_list
jackxu2011 Aug 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/Sender.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ package org.apache.linkis.rpc
import org.apache.linkis.DataWorkCloudApplication
import org.apache.linkis.common.ServiceInstance
import org.apache.linkis.rpc.conf.RPCConfiguration
import org.apache.linkis.rpc.sender.SpringMVCRPCSender
import org.apache.linkis.rpc.utils.RPCUtils

import java.util
Expand Down Expand Up @@ -92,7 +91,7 @@ object Sender {
RPCConfiguration.PUBLIC_SERVICE_APPLICATION_NAME.getValue
)
}
if (!serviceInstanceToSenders.containsKey(serviceInstance))
if (!serviceInstanceToSenders.containsKey(serviceInstance)) {
serviceInstanceToSenders synchronized {
if (!serviceInstanceToSenders.containsKey(serviceInstance)) {
serviceInstanceToSenders.put(
Expand All @@ -101,6 +100,7 @@ object Sender {
)
}
}
}
serviceInstanceToSenders.get(serviceInstance)
}

Expand Down
4 changes: 2 additions & 2 deletions linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import org.reflections.scanners.{
object RPCConfiguration {

val BDP_RPC_BROADCAST_THREAD_SIZE: CommonVars[Integer] =
CommonVars("wds.linkis.rpc.broadcast.thread.num", new Integer(25))
CommonVars("wds.linkis.rpc.broadcast.thread.num", 25)

val BDP_RPC_EUREKA_SERVICE_REFRESH_INTERVAL: CommonVars[TimeType] =
CommonVars("wds.linkis.rpc.eureka.client.refresh.interval", new TimeType("1s"))
Expand Down Expand Up @@ -63,7 +63,7 @@ object RPCConfiguration {

val PUBLIC_SERVICE_LIST: Array[String] = CommonVars(
"wds.linkis.gateway.conf.publicservice.list",
"query,jobhistory,application,configuration,filesystem,udf,variable,microservice,errorcode,bml,datasource"
"cs,contextservice,data-source-manager,metadataquery,metadatamanager,query,jobhistory,application,configuration,filesystem,udf,variable,microservice,errorcode,bml,datasource"
).getValue.split(",")

val METADATAQUERY_SERVICE_APPLICATION_NAME: CommonVars[String] =
Expand Down
20 changes: 0 additions & 20 deletions linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,26 +87,6 @@
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-engineconn-plugin-core</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-executor-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-engineconn-common</artifactId>
</exclusion>
</exclusions>
</dependency>

</dependencies>

<build>
Expand Down
19 changes: 11 additions & 8 deletions ...ger/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,22 +118,24 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging {
queue.foreach { q =>
val yarnQueueName = (q \ "queueName").asInstanceOf[JString].values
if (yarnQueueName == realQueueName) return Some(q)
else if (realQueueName.startsWith(yarnQueueName + "."))
else if (realQueueName.startsWith(yarnQueueName + ".")) {
return getQueue(getChildQueues(q))
}
}
None
case JObject(queue) =>
if (
queue
.find(_._1 == "queueName")
.exists(_._2.asInstanceOf[JString].values == realQueueName)
) Some(queues)
else {
) {
Some(queues)
} else {
val childQueues = queue.find(_._1 == "childQueues")
if (childQueues.isEmpty) None
else getQueue(childQueues.map(_._2).get)
}
case JNull | JNothing => None
case _ => None
}

def getChildQueues(resp: JValue): JValue = {
Expand Down Expand Up @@ -163,13 +165,14 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging {
queue
.find(_._1 == "queueName")
.exists(_._2.asInstanceOf[JString].values == realQueueName)
) return Some(queues)
else if ((queues \ "queues").toOption.nonEmpty) {
) {
return Some(queues)
} else if ((queues \ "queues").toOption.nonEmpty) {
val matchQueue = getQueueOfCapacity(getChildQueuesOfCapacity(queues))
if (matchQueue.nonEmpty) return matchQueue
}
None
case JNull | JNothing => None
case _ => None
}

def getChildQueuesOfCapacity(resp: JValue) = resp \ "queues" \ "queue"
Expand Down Expand Up @@ -265,7 +268,7 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging {
}
}
appInfoBuffer.toArray
case JNull | JNothing => new Array[YarnAppInfo](0)
case _ => new Array[YarnAppInfo](0)
}
}

Expand Down
28 changes: 0 additions & 28 deletions linkis-dist/package/conf/linkis-ps-cs.properties
View file
Open in desktop

This file was deleted.

37 changes: 0 additions & 37 deletions linkis-dist/package/conf/linkis-ps-data-source-manager.properties
View file
Open in desktop

This file was deleted.

21 changes: 0 additions & 21 deletions linkis-dist/package/conf/linkis-ps-metadataquery.properties
View file
Open in desktop

This file was deleted.

20 changes: 15 additions & 5 deletions linkis-dist/package/conf/linkis-ps-publicservice.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,35 @@
# limitations under the License.
#

#
wds.linkis.server.mdm.service.instance.expire-in-seconds=1800

##restful
wds.linkis.server.restful.scan.packages=org.apache.linkis.jobhistory.restful,org.apache.linkis.variable.restful,org.apache.linkis.configuration.restful,org.apache.linkis.udf.api,org.apache.linkis.filesystem.restful,org.apache.linkis.filesystem.restful,org.apache.linkis.instance.label.restful,org.apache.linkis.metadata.restful.api,org.apache.linkis.cs.server.restful,org.apache.linkis.bml.restful,org.apache.linkis.errorcode.server.restful
wds.linkis.server.restful.scan.packages=org.apache.linkis.cs.server.restful,org.apache.linkis.datasourcemanager.core.restful,org.apache.linkis.metadata.query.server.restful,org.apache.linkis.jobhistory.restful,org.apache.linkis.variable.restful,org.apache.linkis.configuration.restful,org.apache.linkis.udf.api,org.apache.linkis.filesystem.restful,org.apache.linkis.filesystem.restful,org.apache.linkis.instance.label.restful,org.apache.linkis.metadata.restful.api,org.apache.linkis.cs.server.restful,org.apache.linkis.bml.restful,org.apache.linkis.errorcode.server.restful

##mybatis
wds.linkis.server.mybatis.mapperLocations=classpath:org/apache/linkis/jobhistory/dao/impl/*.xml,classpath:org/apache/linkis/variable/dao/impl/*.xml,classpath:org/apache/linkis/configuration/dao/impl/*.xml,classpath:org/apache/linkis/udf/dao/impl/*.xml,classpath:org/apache/linkis/instance/label/dao/impl/*.xml,classpath:org/apache/linkis/metadata/hive/dao/impl/*.xml,org/apache/linkis/metadata/dao/impl/*.xml,classpath:org/apache/linkis/bml/dao/impl/*.xml
wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.configuration.entity,org.apache.linkis.jobhistory.entity,org.apache.linkis.udf.entity,org.apache.linkis.variable.entity,org.apache.linkis.instance.label.entity,org.apache.linkis.manager.entity,org.apache.linkis.metadata.domain,org.apache.linkis.bml.entity
wds.linkis.server.mybatis.BasePackage=org.apache.linkis.jobhistory.dao,org.apache.linkis.variable.dao,org.apache.linkis.configuration.dao,org.apache.linkis.udf.dao,org.apache.linkis.instance.label.dao,org.apache.linkis.metadata.hive.dao,org.apache.linkis.metadata.dao,org.apache.linkis.bml.dao,org.apache.linkis.errorcode.server.dao,org.apache.linkis.publicservice.common.lock.dao
wds.linkis.server.mybatis.mapperLocations=classpath*:org/apache/linkis/cs/persistence/dao/impl/*.xml,classpath:org/apache/linkis/datasourcemanager/core/dao/mapper/*.xml,classpath:org/apache/linkis/jobhistory/dao/impl/*.xml,classpath:org/apache/linkis/variable/dao/impl/*.xml,classpath:org/apache/linkis/configuration/dao/impl/*.xml,classpath:org/apache/linkis/udf/dao/impl/*.xml,classpath:org/apache/linkis/instance/label/dao/impl/*.xml,classpath:org/apache/linkis/metadata/hive/dao/impl/*.xml,org/apache/linkis/metadata/dao/impl/*.xml,classpath:org/apache/linkis/bml/dao/impl/*.xml
wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.cs.persistence.entity,org.apache.linkis.datasourcemanager.common.domain,org.apache.linkis.datasourcemanager.core.vo,org.apache.linkis.configuration.entity,org.apache.linkis.jobhistory.entity,org.apache.linkis.udf.entity,org.apache.linkis.variable.entity,org.apache.linkis.instance.label.entity,org.apache.linkis.manager.entity,org.apache.linkis.metadata.domain,org.apache.linkis.bml.entity
wds.linkis.server.mybatis.BasePackage=org.apache.linkis.cs.persistence.dao,org.apache.linkis.datasourcemanager.core.dao,org.apache.linkis.jobhistory.dao,org.apache.linkis.variable.dao,org.apache.linkis.configuration.dao,org.apache.linkis.udf.dao,org.apache.linkis.instance.label.dao,org.apache.linkis.metadata.hive.dao,org.apache.linkis.metadata.dao,org.apache.linkis.bml.dao,org.apache.linkis.errorcode.server.dao,org.apache.linkis.publicservice.common.lock.dao

##workspace
wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix=/
wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration,org.apache.linkis.entrance.conf.EntranceSpringConfiguration

##hive meta
hive.meta.url=
hive.meta.user=
hive.meta.password=
wds.linkis.metadata.hive.encode.enabled=false

# associated with the logged-in user when querying metadata:default value is true
#linkis.metadata.hive.permission.with-login-user-enabled
#wds.linkis.jobhistory.undone.job.minimum.id=0
#wds.linkis.jobhistory.undone.job.refreshtime.daily=00:15
wds.linkis.cs.deserialize.replace_package_header.enable=false

##Spring
spring.server.port=9105
spring.spring.main.allow-bean-definition-overriding=true
spring.spring.jackson.serialization.FAIL_ON_EMPTY_BEANS=false
# ps-cs prefix must be started with 'cs_'
spring.eureka.instance.metadata-map.route=cs_1_dev
5 changes: 1 addition & 4 deletions linkis-dist/package/conf/linkis.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,11 @@ wds.linkis.bml.is.hdfs=true
wds.linkis.home=/appcom/Install/LinkisInstall
#Linkis governance station administrators
wds.linkis.governance.station.admin=hadoop
wds.linkis.gateway.conf.publicservice.list=query,jobhistory,application,configuration,filesystem,udf,variable,microservice,errorcode,bml,datasource
#wds.linkis.gateway.conf.publicservice.list=query,jobhistory,application,configuration,filesystem,udf,variable,microservice,errorcode,bml,datasource

#wds.linkis.prometheus.enable=true
wds.linkis.server.user.restful.uri.pass.auth=/actuator/prometheus,/api/rest_j/v1/offline,/api/rest_j/v1/doc.html,/api/rest_j/v1/swagger-resources,/api/rest_j/v1/webjars,/api/rest_j/v1/v2/api-docs




wds.linkis.gateway.conf.metadataquery.list=metadatamanager,metadataquery
spring.spring.servlet.multipart.max-file-size=500MB
spring.spring.servlet.multipart.max-request-size=500MB
Expand Down
33 changes: 0 additions & 33 deletions linkis-dist/package/sbin/ext/linkis-ps-cs
View file
Open in desktop

This file was deleted.

35 changes: 0 additions & 35 deletions linkis-dist/package/sbin/ext/linkis-ps-data-source-manager
View file
Open in desktop

This file was deleted.

33 changes: 0 additions & 33 deletions linkis-dist/package/sbin/ext/linkis-ps-metadataquery
View file
Open in desktop

This file was deleted.

Loading