-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Description

flinkx run with flink on yarn perjob mode, the command and job config as follows. it does not throw an exception, and blocked in YarnClusterDescriptor.deployJobCluster. what is the reason of this probolem.
bin/flinkx -mode yarnPer -job ./job/demo.json -pluginRoot /home/yifanzhang/flinkx/plugins -flinkconf /home/yifanzhang/flink-1.8.1/conf -yarnconf /opt/cloudera/parcels/CDH/lib/hadoop-yarn/etc/hadoop -flinkLibJar /home/yifanzhang/flink/lib
{
"job" : {
"content" : [ {
"reader" : {
"parameter" : {
"groupId" : "default",
"topic" : "test",
"consumerSettings" : {
"zookeeper.connect" : "10.126.144.37:2181,10.126.144.38:2181,10.126.144.39:2181/cp3",
"bootstrap.servers" : "10.126.144.34:9092,10.126.144.35:9092,10.126.144.36:9092",
"auto.commit.interval.ms" : "1000",
"auto.offset.reset" : "latest"
}
},
"name" : "kafka11reader",
"type" : 14
},
"writer" : {
"parameter" : {
"path" : "/tmp",
"fileName" : "test",
"hadoopConfig" : {
"dfs.ha.namenodes.ns1" : "namenode252,namenode369",
"fs.defaultFS" : "hdfs:///idc-nn",
"dfs.namenode.rpc-address.idc-nn.namenode252" : "shoyi04plpebdp015.mcd.com.cn:8020",
"dfs.client.failover.proxy.provider.idc-nn" : "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"dfs.namenode.rpc-address.idc-nn.namenode369" : "shoyi05plpebdp019.mcd.com.cn:8022",
"dfs.nameservices" : "idc-nn",
"fs.hdfs.impl.disable.cache" : "true",
"fs.hdfs.impl" : "org.apache.hadoop.hdfs.DistributedFileSystem",
"hadoop.security.authorization": "true",
"hadoop.security.authentication": "Kerberos",
"dfs.namenode.kerberos.principal": "yifanzhang",
"dfs.namenode.keytab.file": "/home/yifanzhang/yifanzhang.keytab",
"java.security.krb5.conf": "/etc/krb5.conf"
},
"column" : [ {
"name" : "id",
"index" : 0,
"type" : "string"
} ],
"defaultFS" : "hdfs:///idc-nn",
"writeMode" : "APPEND",
"fieldDelimiter" : ",",
"fileType" : "orc",
"charsetName" : "utf-8"
},
"name" : "hdfswriter",
"type" : 6
}
} ],
"setting" : {
"restore" : {
"isRestore" : true,
"isStream" : true
},
"errorLimit" : {
},
"speed" : {
"bytes" : 2097152,
"channel" : 3
}
}
}
}