python实现采集yarn队列资源使用百分比，暴露为prometheus的exporter格式，进行yarn队列资源监控

python脚本实现，访问官方提供的地址，获取队列资源使用指标，并将指标进行json解析，之后再转换为prometheus认识的数据格式，暴露于端口下
#coding=utf-8

'''
通过访问官方提供的yarn restful api界面,获取yarn资源指标
分析json取的所需指标
以pormetheus的数据格式将指标暴露在指定端口下

入参：yarn的ip:port export暴露的机器IP(建议为本机IP) export暴露的端口号
'''

import prometheus_client
from prometheus_client import Gauge
from prometheus_client.core import CollectorRegistry
from flask import Response, Flask

import json
import urllib.request
import sys


hjson = json.loads('{}')

app = Flask(__name__)

REGISTRY = CollectorRegistry(auto_describe=False)

usememory_percentage_temp = Gauge(
    name='vcores_percentage',
    documentation= 'vcores_percentage',
    labelnames=['queue_name'],
    registry=REGISTRY)
usevCores_percentage_temp = Gauge(
   'memory_percentage',
    'memory_percentage',
    labelnames=['queue_name'],
    registry=REGISTRY)

def getMetric(ipport):

    html = urllib.request.urlopen(r'http://'+ipport+'/ws/v1/cluster/scheduler')

    data = html.read().decode("utf-8")

    global hjson
    hjson = json.loads(data)
    queuelist = []
    coresList = []
    memList = []

  # 不同版本，json格式不一样，需要具体解析
    for i in range(0,len(hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'])):
        queuename = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['queueName']
        usevCores = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['usedResources']['vCores']
        usermemory = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['usedResources']['memory']
        maxvCores = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['maxResources']['vCores']
        maxmemory = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['maxResources']['memory']
        usermemory_percentage = 0.0000
        usevCores_percentage = 0.0000
        if(maxvCores != 0):
            usevCores_percentage = round(usevCores/maxvCores,4)
        if(maxmemory != 0):
            usermemory_percentage = round(usermemory/maxmemory,4)
        queuelist.append(queuename.split(".")[1])
        coresList.append(float(usevCores_percentage))
        memList.append(float(usermemory_percentage))

    for i in range(0,len(hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'])):
        usememory_percentage_temp.labels(queue_name=str(queuelist[i])).set(float(memList[i]))
        usevCores_percentage_temp.labels(queue_name=str(queuelist[i])).set(float(coresList[i]))





@app.route('/metrics')
def r_value():
    return Response(prometheus_client.generate_latest(REGISTRY),
                    mimetype="text/plain")


@app.route('/')
def index():
    return "yarn metric address: ip:port/metrics"



if __name__ == "__main__":
	#yarn的ip和端口
    # yarn_ipport = str(sys.argv[1])
    #展示的ip，建议本机ip
    # showIp = str(sys.argv[2])
    #展示的端口
    # showPort = int(sys.argv[3])

	#yarn的ip和端口
    yarn_ipport = "127.0.0.1:8001"
    #展示的ip，建议本机ipx
    showIp = "localhost"
    #展示的端口
    showPort = 8000
    getMetric(yarn_ipport)

    app.run(host=showIp,port=showPort,debug=True)

效果图如下