Class cascading.DistanceUseCasePlatformTest

2016-01-14 17:40:11,732 INFO  platform.PlatformRunner (PlatformRunner.java:getPlatformClassesFromClasspath(195)) - found 1 test platforms from classpath
2016-01-14 17:40:11,766 INFO  platform.PlatformRunner (PlatformRunner.java:addPlatform(244)) - adding test: cascading.DistanceUseCasePlatformTest, with platform: hadoop2-tez
2016-01-14 17:40:11,801 INFO  tez.Hadoop2TezPlatform (Hadoop2TezPlatform.java:setUp(136)) - not using cluster
2016-01-14 17:40:12,374 WARN  util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(62)) - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2016-01-14 17:40:12,683 INFO  platform.TestPlatform (TestPlatform.java:getGlobalProperties(73)) - platform property overrides: 
2016-01-14 17:40:12,927 INFO  util.Util (Util.java:findMainClass(1327)) - using default application jar, may cause class not found exceptions on the cluster
2016-01-14 17:40:12,929 INFO  planner.Hadoop2TezPlanner (Hadoop2TezPlanner.java:initialize(149)) - using application jar: /opt/concurrent/home/teamcity/.gradle/caches/modules-2/files-2.1/cascading/cascading-hadoop2-tez/3.0.4/561447ee876073bb61fddc52e2ad157f45c8edd/cascading-hadoop2-tez-3.0.4.jar
2016-01-14 17:40:13,050 INFO  property.AppProps (AppProps.java:getAppID(177)) - using app.id: 47E0B5BF0E5547968CEDE7DA745E828C
2016-01-14 17:40:13,318 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] executed rule registry: NoHashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.155
2016-01-14 17:40:13,377 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] executed rule registry: HashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.215
2016-01-14 17:40:13,379 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: NoHashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:13,380 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: HashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:13,382 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: NoHashJoinHadoop2TezRuleRegistry, result was selected using: 'default comparator: selects plan with fewest steps and fewest nodes'
2016-01-14 17:40:13,555 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt
2016-01-14 17:40:13,753 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceComposite/euclidean/composite
2016-01-14 17:40:14,112 INFO  util.Version (Version.java:printBanner(82)) - Concurrent, Inc - Cascading 3.0.4
2016-01-14 17:40:14,114 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] starting
2016-01-14 17:40:14,114 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:14,115 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  sink: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceComposite/euclidean/composite"]
2016-01-14 17:40:14,115 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  parallel execution of steps is enabled: false
2016-01-14 17:40:14,115 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  executing total steps: 1
2016-01-14 17:40:14,115 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  allocating management threads: 1
2016-01-14 17:40:14,116 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] starting step: (1/1) ...osite/euclidean/composite
2016-01-14 17:40:14,116 WARN  flow.Flow (BaseFlow.java:logWarn(1445)) - [euclidean] 'yarn.timeline-service.enabled' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon
2016-01-14 17:40:14,135 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tez session mode enabled: true
2016-01-14 17:40:14,297 INFO  client.TezClient (TezClient.java:<init>(153)) - Tez Client Version: [ component=tez-api, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:14,303 INFO  client.TezClient (TezClient.java:start(333)) - Session mode. Starting session.
2016-01-14 17:40:14,303 INFO  client.TezClientUtils (TezClientUtils.java:setupTezJarsLocalResources(159)) - Ignoring 'tez.lib.uris' since  'tez.ignore.lib.uris' is set to true
2016-01-14 17:40:14,351 INFO  client.TezClient (TezCommonUtils.java:createTezSystemStagingPath(118)) - Tez system stage directory file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001 doesn't exist and is created
2016-01-14 17:40:14,405 INFO  client.LocalClient (LocalClient.java:createDAGAppMaster(328)) - DAGAppMaster thread has been created
2016-01-14 17:40:14,405 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:14,406 INFO  client.LocalClient (LocalClient.java:run(275)) - Using staging directory: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001
2016-01-14 17:40:14,446 INFO  app.DAGAppMaster (DAGAppMaster.java:<init>(309)) - Created DAGAppMaster for application appattempt_1452793214302_0001_000000, versionInfo=[ component=tez-dag, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:14,486 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(333)) - Comparing client version with AM version, clientVersion=0.6.2, AMVersion=0.6.2
2016-01-14 17:40:14,501 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(386)) - Adding session token to jobTokenSecretManager for application
2016-01-14 17:40:14,506 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: NEW
2016-01-14 17:40:14,506 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(238)) - DAGAppMaster is not started wait for 100ms...
2016-01-14 17:40:14,507 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:14,525 INFO  node.AMNodeTracker (AMNodeTracker.java:serviceInit(77)) - blacklistDisablePercent is 33, blacklistingEnabled: false, maxTaskFailuresPerNode: 1
2016-01-14 17:40:14,528 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceInit(50)) - Initializing HistoryEventHandler
2016-01-14 17:40:14,532 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(66)) - Log file location for SimpleHistoryLoggingService not specified, defaulting to containerLogDir=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001/localmode-log-dir
2016-01-14 17:40:14,532 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(88)) - Initializing SimpleHistoryLoggingService, logFileLocation=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001/localmode-log-dir/history.txt.appattempt_1452793214302_0001_000000, maxErrors=10
2016-01-14 17:40:14,532 INFO  recovery.RecoveryService (RecoveryService.java:serviceInit(109)) - Initializing RecoveryService
2016-01-14 17:40:14,534 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_LAUNCHED]: appAttemptId=appattempt_1452793214302_0001_000000, appSubmitTime=1452793214432, launchTime=1452793214445
2016-01-14 17:40:14,540 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStart(73)) - Starting HistoryEventHandler
2016-01-14 17:40:14,541 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStart(95)) - Starting SimpleHistoryLoggingService
2016-01-14 17:40:14,542 INFO  recovery.RecoveryService (RecoveryService.java:serviceStart(127)) - Starting RecoveryService
2016-01-14 17:40:14,542 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_LAUNCHED to history file
2016-01-14 17:40:14,607 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: INITED
2016-01-14 17:40:14,608 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(240)) - DAGAppMaster is not startetd wait for 100ms...
2016-01-14 17:40:14,608 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:14,629 INFO  client.DAGClientServer (DAGClientServer.java:serviceStart(91)) - Instantiated DAGClientRPCServer at ip-10-187-41-40.ec2.internal/10.187.41.40:60474
2016-01-14 17:40:14,657 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-dagappmaster.properties,hadoop-metrics2.properties
2016-01-14 17:40:14,708 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: INITED
2016-01-14 17:40:14,708 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(240)) - DAGAppMaster is not startetd wait for 100ms...
2016-01-14 17:40:14,709 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:14,733 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_STARTED to history file
2016-01-14 17:40:14,734 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_STARTED]: appAttemptId=appattempt_1452793214302_0001_000000, startTime=1452793214733
2016-01-14 17:40:14,734 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceStart(1644)) - In Session mode. Waiting for DAG over RPC
2016-01-14 17:40:14,809 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: IDLE
2016-01-14 17:40:14,825 INFO  client.TezClient (TezClient.java:start(367)) - The url to track the Tez Session: N/A
2016-01-14 17:40:14,825 INFO  client.TezClient (TezClient.java:submitDAGSession(405)) - Submitting dag to TezSession, sessionName=(1/1) ...osite/euclidean/composite, applicationId=application_1452793214302_0001, dagName=[D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite
2016-01-14 17:40:15,045 INFO  app.DAGAppMaster (DAGAppMaster.java:submitDAGToAppMaster(1126)) - Starting DAG submitted via RPC: [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite
2016-01-14 17:40:15,148 INFO  app.DAGAppMaster (DAGAppMaster.java:generateDAGVizFile(870)) - Generating DAG graphviz file, dagId=dag_1452793214302_0001_1, filePath=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001/localmode-log-dir/dag_1452793214302_0001_1.dot
2016-01-14 17:40:15,152 INFO  app.DAGAppMaster (DAGAppMaster.java:writePBTextFile(887)) - Writing DAG plan to: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001/localmode-log-dir/dag_1452793214302_0001_1-tez-dag.pb.txt
2016-01-14 17:40:15,251 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793214302_0001_1
2016-01-14 17:40:15,251 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:15,251 INFO  app.DAGAppMaster (DAGAppMaster.java:startDAG(2051)) - Running DAG: [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite
2016-01-14 17:40:15 Running Dag: dag_1452793214302_0001_1
2016-01-14 17:40:15,279 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_SUBMITTED to history file
2016-01-14 17:40:15,279 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:DAG_SUBMITTED]: dagID=dag_1452793214302_0001_1, submitTime=1452793215045
2016-01-14 17:40:15,325 INFO  impl.VertexImpl (VertexImpl.java:setAdditionalOutputs(4033)) - setting additional outputs for vertex A56BB627E00347C6901A0CD066814E35
2016-01-14 17:40:15,336 INFO  impl.DAGImpl (DAGImpl.java:assignDAGScheduler(1381)) - Using DAG Scheduler: org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
2016-01-14 17:40:15,340 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:DAG_INITIALIZED]: dagID=dag_1452793214302_0001_1, initTime=1452793215280
2016-01-14 17:40:15,341 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793214302_0001_1 transitioned from NEW to INITED due to event DAG_INIT
2016-01-14 17:40:15,342 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_INITIALIZED to history file
2016-01-14 17:40:15,343 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:DAG_STARTED]: dagID=dag_1452793214302_0001_1, startTime=1452793215342
2016-01-14 17:40:15,344 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_STARTED to history file
2016-01-14 17:40:15,344 INFO  impl.DAGImpl (DAGImpl.java:transition(1614)) - Added additional resources : [[]] to classpath
2016-01-14 17:40:15,348 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793214302_0001_1 transitioned from INITED to RUNNING due to event DAG_START
2016-01-14 17:40:15,349 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2068)) - Root Inputs exist for Vertex: A0CAF2DFE79A40B18A23D3DFAAA8C95A : {899CE115164C436FAEBD3FBD4572AF81={InputName=899CE115164C436FAEBD3FBD4572AF81}, {Descriptor=ClassName=org.apache.tez.mapreduce.input.MRInput, hasPayload=true}, {ControllerDescriptor=ClassName=org.apache.tez.mapreduce.common.MRInputAMSplitGenerator, hasPayload=false}}
2016-01-14 17:40:15,350 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2078)) - Starting root input initializer for input: 899CE115164C436FAEBD3FBD4572AF81, with class: [org.apache.tez.mapreduce.common.MRInputAMSplitGenerator]
2016-01-14 17:40:15,350 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2191)) - Setting vertexManager to RootInputVertexManager for vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,356 INFO  client.TezClient (TezClient.java:submitDAGSession(478)) - Submitted dag to TezSession, sessionName=(1/1) ...osite/euclidean/composite, applicationId=application_1452793214302_0001, dagName=[D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite
2016-01-14 17:40:15,359 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2983)) - Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split to set #tasks for the vertex vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,359 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2987)) - Vertex will initialize from input initializer. vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,363 INFO  impl.VertexImpl (VertexImpl.java:setupInputInitializerManager(3933)) - Vertex will initialize via inputInitializers vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]. Starting root input initializers: 1
2016-01-14 17:40:15,379 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:run(242)) - Starting InputInitializer for Input: 899CE115164C436FAEBD3FBD4572AF81 on vertex vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,389 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] transitioned from NEW to INITIALIZING due to event V_INIT
2016-01-14 17:40:15,394 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:15,396 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] submitted tez dag to app master: application_1452793214302_0001, with dag id: dag_1452793214302_0001_1
2016-01-14 17:40:15,408 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(92)) - Emitting serialized splits: true
2016-01-14 17:40:15,411 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(110)) - Input 899CE115164C436FAEBD3FBD4572AF81 asking for 1 tasks. Headroom: 672 Task Resource: 1024 waves: 1.7
2016-01-14 17:40:15,413 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(121)) - Grouping input splits
2016-01-14 17:40:15,414 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(294)) - Generating mapred api input splits
2016-01-14 17:40:15,429 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:15,429 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:15,448 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:15,451 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:15,451 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_INITIALIZED]: vertexName=D92AEA6F7A9C4306A0CF848338AAF176, vertexId=vertex_1452793214302_0001_1_01, initRequestedTime=1452793215390, initedTime=1452793215450, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:15,452 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:15,456 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:15,471 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(300)) - NumSplits: 1, SerializedSize: 404
2016-01-14 17:40:15,475 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:onSuccess(272)) - Succeeded InputInitializer for Input: 899CE115164C436FAEBD3FBD4572AF81 on vertex vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,481 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:15,481 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:15,481 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:15,482 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:15,482 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_INITIALIZED]: vertexName=A0935742873842248EF1E3D5DBA6114F, vertexId=vertex_1452793214302_0001_1_02, initRequestedTime=1452793215456, initedTime=1452793215482, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:15,483 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:15,483 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] numTasks: -1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:15,484 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1374)) - Got updated RootInputsSpecs: {899CE115164C436FAEBD3FBD4572AF81=forAllWorkUnits=true, update=[1]}
2016-01-14 17:40:15,484 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1393)) - Vertex vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] parallelism set to 1
2016-01-14 17:40:15,485 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] numTasks: 1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:15,503 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793214302_0001_1_00, eventCount=1
2016-01-14 17:40:15,503 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:15,503 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_INITIALIZED]: vertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, vertexId=vertex_1452793214302_0001_1_00, initRequestedTime=1452793215349, initedTime=1452793215503, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=1
2016-01-14 17:40:15,506 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] with distanceFromRoot: 0
2016-01-14 17:40:15,506 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] transitioned from INITIALIZING to INITED due to event V_ROOT_INPUT_INITIALIZED
2016-01-14 17:40:15,506 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:15,532 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:15,532 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:15,532 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:15,533 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1915)) - Invoking committer inits for vertex, vertexId=vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:15,533 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1927)) - Instantiating committer for output=069C4602D28844528CDE09F3C716F571, vertexId=vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35], committerClass=org.apache.tez.mapreduce.committer.MROutputCommitter
2016-01-14 17:40:15,536 INFO  impl.VertexImpl (VertexImpl.java:run(1945)) - Invoking committer init for output=069C4602D28844528CDE09F3C716F571, vertexId=vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:15,546 INFO  committer.MROutputCommitter (MROutputCommitter.java:getOutputCommitter(145)) - OutputCommitter for outputName=069C4602D28844528CDE09F3C716F571, vertexName=A56BB627E00347C6901A0CD066814E35, outputCommitterClass=org.apache.hadoop.mapred.FileOutputCommitter
2016-01-14 17:40:15,555 INFO  impl.VertexImpl (VertexImpl.java:run(1949)) - Invoking committer setup for output=069C4602D28844528CDE09F3C716F571, vertexId=vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:15,557 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_INITIALIZED]: vertexName=A56BB627E00347C6901A0CD066814E35, vertexId=vertex_1452793214302_0001_1_03, initRequestedTime=1452793215506, initedTime=1452793215557, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:15,557 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:15,557 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:15,558 INFO  impl.ImmediateStartVertexManager (ImmediateStartVertexManager.java:scheduleTasks(99)) - Starting 1 in A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:15,558 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,560 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:15,560 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793214302_0001_1_00, startRequestedTime=1452793215558, startedTime=1452793215558
2016-01-14 17:40:15,560 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:15,581 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:15,581 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_STARTED]: vertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, taskId=task_1452793214302_0001_1_00_000000, scheduledTime=1452793215580, launchTime=1452793215580
2016-01-14 17:40:15,581 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_00_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:15,582 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793214302_0001_1_00 for vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] numStartedSources: 1 numSources: 1
2016-01-14 17:40:15,582 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] with distanceFromRoot: 1
2016-01-14 17:40:15,583 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:15,588 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: A0CAF2DFE79A40B18A23D3DFAAA8C95A in vertex: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:15,590 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: D92AEA6F7A9C4306A0CF848338AAF176 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:15,590 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:15,590 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793214302_0001_1_01, startRequestedTime=1452793215584, startedTime=1452793215584
2016-01-14 17:40:15,590 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:15,592 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: A0CAF2DFE79A40B18A23D3DFAAA8C95A, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=899CE115164C436FAEBD3FBD4572AF81, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=D92AEA6F7A9C4306A0CF848338AAF176, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:15,594 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_00_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:15,595 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793214302_0001_1_01 for vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] numStartedSources: 1 numSources: 1
2016-01-14 17:40:15,595 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] with distanceFromRoot: 2
2016-01-14 17:40:15,595 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:15,595 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: D92AEA6F7A9C4306A0CF848338AAF176 in vertex: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:15,596 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: A0935742873842248EF1E3D5DBA6114F with 1 source tasks and 1 pending tasks
2016-01-14 17:40:15,596 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:15,596 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793214302_0001_1_02, startRequestedTime=1452793215595, startedTime=1452793215595
2016-01-14 17:40:15,596 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:15,596 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793214302_0001_1_02 for vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] numStartedSources: 1 numSources: 1
2016-01-14 17:40:15,597 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] with distanceFromRoot: 3
2016-01-14 17:40:15,597 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: A0935742873842248EF1E3D5DBA6114F in vertex: A56BB627E00347C6901A0CD066814E35
2016-01-14 17:40:15,597 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: A56BB627E00347C6901A0CD066814E35 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:15,597 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:15,597 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793214302_0001_1_03, startRequestedTime=1452793215597, startedTime=1452793215597
2016-01-14 17:40:15,598 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:15,633 INFO  node.AMNodeTracker (AMNodeTracker.java:nodeSeen(91)) - Adding new node: 127.0.0.1:0
2016-01-14 17:40:15,636 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(112)) - Adding #1 tokens and #0 secret keys for NM use for launching container
2016-01-14 17:40:15,636 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(122)) - Putting shuffle token in serviceData
2016-01-14 17:40:15,640 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793214302_0001_00_000001
2016-01-14 17:40:15,640 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:15,649 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793214302_0001_00_000001
2016-01-14 17:40:15,649 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - DAGAppMaster metrics system already initialized!
2016-01-14 17:40:15,655 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:15,655 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:15,655 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793214302_0001_00_000001, launchTime=1452793215655
2016-01-14 17:40:15,660 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000001
2016-01-14 17:40:15,663 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793214302_0001_1_00_000000_0] to container: [container_1452793214302_0001_00_000001]
2016-01-14 17:40:15,664 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:15,665 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793214302_0001_00_000001 given task: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:15,665 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000001: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:15 Starting to run new task attempt: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:15,666 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:15,667 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:15,667 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793214302_0001_1_00_000000_0] started. Is using containerId: [container_1452793214302_0001_00_000001] on NM: [127.0.0.1:0]
2016-01-14 17:40:15,668 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, taskAttemptId=attempt_1452793214302_0001_1_00_000000_0, startTime=1452793215665, containerId=container_1452793214302_0001_00_000001, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793214302_0001_00_000001/teamcity, completedLogs=
2016-01-14 17:40:15,669 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:15,669 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_00_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:15,670 INFO  counters.Limits (Limits.java:ensureInitialized(59)) - Counter limits initialized with parameters:  GROUP_NAME_MAX=128, MAX_GROUPS=500, COUNTER_NAME_MAX=64, MAX_COUNTERS=1200
2016-01-14 17:40:15,672 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:15,672 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:15,673 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:15,675 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_00_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:15,676 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:15,697 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:15,697 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: A0CAF2DFE79A40B18A23D3DFAAA8C95A, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=899CE115164C436FAEBD3FBD4572AF81, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=D92AEA6F7A9C4306A0CF848338AAF176, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:15,700 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:15,704 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:15,705 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:15,705 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:15,706 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793214302_0001_1_00_000000_0 sent events: (0-1)
2016-01-14 17:40:15,713 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=899CE115164C436FAEBD3FBD4572AF81, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }
2016-01-14 17:40:15,713 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:15,713 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=D92AEA6F7A9C4306A0CF848338AAF176, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:15,716 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:15,716 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:15,717 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,721 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: INPUT, componentVertexName: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,722 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:15,734 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:15,735 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:15,737 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:15,745 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:15,755 INFO  input.MRInput (MRInput.java:initialize(440)) - Using New mapreduce API: false, split information via event: true
2016-01-14 17:40:15,755 INFO  input.MRInput (MRInput.java:initializeInternal(493)) - Initialzed MRInput: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,755 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,773 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: A0CAF2DFE79A40B18A23D3DFAAA8C95A, ordinal: 0
2016-01-14 17:40:15,774 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A0CAF2DFE79A40B18A23D3DFAAA8C95A, mem on start (mb), free: 183, total: 281, max: 672
2016-01-14 17:40:15,774 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:15,774 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:15,774 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:15,774 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:15,776 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:15,781 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.input.MRInput
2016-01-14 17:40:15,782 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:15,782 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:15,783 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:15,784 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 104857600, TotalRequestedScaled: 9.679163076923077E7, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.15
2016-01-14 17:40:15,784 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, 899CE115164C436FAEBD3FBD4572AF81, org.apache.tez.mapreduce.input.MRInput: requested=0, allocated=0
2016-01-14 17:40:15,785 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, D92AEA6F7A9C4306A0CF848338AAF176, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=104857600
2016-01-14 17:40:15,785 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:15,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: 899CE115164C436FAEBD3FBD4572AF81 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:15,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,790 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:15,790 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:15,790 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:15,791 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:15,794 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:15,796 INFO  input.MRInput (MRInput.java:initFromEventInternal(619)) - Initializing RecordReader from event
2016-01-14 17:40:15,806 INFO  input.MRInput (MRInput.java:initFromEventInternal(630)) - Split Details -> SplitClass: org.apache.hadoop.mapred.split.TezGroupedSplit, OldSplit: org.apache.hadoop.mapred.split.TezGroupedSplit@3ab65057
2016-01-14 17:40:15,819 INFO  lib.MRReaderMapred (MRReaderMapred.java:setIncrementalConfigParams(163)) - Processing split: org.apache.hadoop.mapred.split.TezGroupedSplit@3ab65057
2016-01-14 17:40:15,820 INFO  input.MRInput (MRInput.java:initFromEventInternal(634)) - Initialized RecordReader from event
2016-01-14 17:40:15,820 INFO  input.MRInput (MRInput.java:processSplitEvent(586)) - Notifying on RecordReader Initialized
2016-01-14 17:40:15,848 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:15,896 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"] streamed: true, id: 899CE115164C436FAEBD3FBD4572AF81
2016-01-14 17:40:15,897 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: CoGroup(euclidean)[by:[{1}:'l'][numSelfJoins:1]], id: 9F815B2E169043A38A913571AA935F90
2016-01-14 17:40:15,898 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: CoGroup(euclidean)[by:[{1}:'l'][numSelfJoins:1]] 9F815B2E169043A38A913571AA935F90
2016-01-14 17:40:15,927 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@5fda19c9; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@67d30c48; comparator=cascading.tuple.hadoop.util.TupleComparator@50cb8038; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:15,930 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:15,933 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:15,934 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:16,065 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 26214396(104857584)
2016-01-14 17:40:16,066 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 100
2016-01-14 17:40:16,066 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 83886080
2016-01-14 17:40:16,066 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 104857600
2016-01-14 17:40:16,066 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 26214396; length = 6553600
2016-01-14 17:40:16,076 INFO  element.TezSourceStage (TezSourceStage.java:prepare(63)) - calling MRInput#start() on: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:16,076 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:16,079 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: A0CAF2DFE79A40B18A23D3DFAAA8C95A, all 1 inputs ready in: 00:00:00.001
2016-01-14 17:40:16,090 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:16,091 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A0CAF2DFE79A40B18A23D3DFAAA8C95A, mem on close (mb), free: 70, total: 281, max: 672
2016-01-14 17:40:16,091 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:16,091 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:16,091 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 1676; bufvoid = 104857600
2016-01-14 17:40:16,092 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 26214396(104857584); kvend = 26214260(104857040); length = 137/6553600
2016-01-14 17:40:16,111 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:16,131 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793214302_0001_1_00_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:16,132 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793214302_0001_1_00, eventCount=1
2016-01-14 17:40:16,135 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 27 [[File System Counters FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0][cascading.flow.SliceCounters Process_Begin_Time=1452793216080, Process_Duration=10, Process_End_Time=1452793216090, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2][cascading.flow.StepCounters Tuples_Read=7]]
2016-01-14 17:40:16,135 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:16,136 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:16,139 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,141 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000001
2016-01-14 17:40:16,147 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, taskAttemptId=attempt_1452793214302_0001_1_00_000000_0, startTime=1452793215665, finishTime=1452793216133, timeTaken=468, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216080, Process_Duration=10, Process_End_Time=1452793216090, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:16,150 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_00_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:16,150 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:16,154 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:16,156 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:16,165 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_FINISHED]: vertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, taskId=task_1452793214302_0001_1_00_000000, startTime=1452793215665, finishTime=1452793216155, timeTaken=490, status=SUCCEEDED, successfulAttemptID=attempt_1452793214302_0001_1_00_000000_0, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216080, Process_Duration=10, Process_End_Time=1452793216090, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:16,166 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_00_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:16,168 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:16,168 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] : 1
2016-01-14 17:40:16,169 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:16,174 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:16,179 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_FINISHED]: vertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, vertexId=vertex_1452793214302_0001_1_00, initRequestedTime=1452793215349, initedTime=1452793215503, startRequestedTime=1452793215558, startedTime=1452793215558, finishTime=1452793216169, timeTaken=611, status=SUCCEEDED, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216080, Process_Duration=10, Process_End_Time=1452793216090, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2, cascading.flow.StepCounters, Tuples_Read=7, vertexStats=firstTaskStartTime=1452793215665, firstTasksToStart=[ task_1452793214302_0001_1_00_000000 ], lastTaskFinishTime=1452793216133, lastTasksToFinish=[ task_1452793214302_0001_1_00_000000 ], minTaskDuration=468, maxTaskDuration=468, avgTaskDuration=468.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793214302_0001_1_00_000000 ], longestDurationTasks=[ task_1452793214302_0001_1_00_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:16,179 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:16,181 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:16,182 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793214302_0001_00_000001, stoppedTime=1452793216181, exitStatus=0
2016-01-14 17:40:16,186 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:16,187 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] attempt: attempt_1452793214302_0001_1_00_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:16,187 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,187 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:16,189 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A] completed., numCompletedVertices=1, numSuccessfulVertices=1, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:16,190 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793214302_0001_00_000001. Relying on regular task shutdown for it to end
2016-01-14 17:40:16,190 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_STARTED]: vertexName=D92AEA6F7A9C4306A0CF848338AAF176, taskId=task_1452793214302_0001_1_01_000000, scheduledTime=1452793216190, launchTime=1452793216190
2016-01-14 17:40:16,190 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:16,191 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_01_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:16,191 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:16,191 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:16,192 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: D92AEA6F7A9C4306A0CF848338AAF176, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A0935742873842248EF1E3D5DBA6114F, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:16,192 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_01_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:16,192 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:16,194 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793214302_0001_00_000002
2016-01-14 17:40:16,194 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000002 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:16,198 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793214302_0001_00_000002
2016-01-14 17:40:16,198 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - DAGAppMaster metrics system already initialized!
2016-01-14 17:40:16,200 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000002 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:16,200 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:16,201 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793214302_0001_00_000002, launchTime=1452793216199
2016-01-14 17:40:16,202 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000002
2016-01-14 17:40:16,203 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793214302_0001_1_01_000000_0] to container: [container_1452793214302_0001_00_000002]
2016-01-14 17:40:16,204 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000002 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:16,205 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793214302_0001_1_01_000000_0] started. Is using containerId: [container_1452793214302_0001_00_000002] on NM: [127.0.0.1:0]
2016-01-14 17:40:16,205 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=D92AEA6F7A9C4306A0CF848338AAF176, taskAttemptId=attempt_1452793214302_0001_1_01_000000_0, startTime=1452793216204, containerId=container_1452793214302_0001_00_000002, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793214302_0001_00_000002/teamcity, completedLogs=
2016-01-14 17:40:16,205 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_01_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:16,206 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_01_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:16,206 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:16,206 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793214302_0001_00_000002 given task: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,207 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000002: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16 Starting to run new task attempt: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,208 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793214302_0001_00_000001 is valid, but no longer registered, and will be killed
2016-01-14 17:40:16,208 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000001: 62 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:16,208 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793214302_0001_00_000001, Exiting
2016-01-14 17:40:16,208 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793214302_0001_00_000001
2016-01-14 17:40:16,208 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793214302_0001_00_000001
2016-01-14 17:40:16,208 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,209 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,209 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:16,209 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:16,210 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:16,210 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:16,211 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:16,212 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: D92AEA6F7A9C4306A0CF848338AAF176, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A0935742873842248EF1E3D5DBA6114F, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:16,212 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:16,220 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,227 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:16,227 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,224 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793214302_0001_00_000001 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:16,228 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793214302_0001_00_000001 completed successfully
2016-01-14 17:40:16,230 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=A0CAF2DFE79A40B18A23D3DFAAA8C95A, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:16,230 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793214302_0001_1_01_000000_0 sent events: (0-1)
2016-01-14 17:40:16,230 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,231 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=A0935742873842248EF1E3D5DBA6114F, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:16,232 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000001 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:16,234 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:16,235 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:16,235 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,238 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,242 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:16,243 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:16,243 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,246 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,267 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:16,268 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:16,269 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: D92AEA6F7A9C4306A0CF848338AAF176, ordinal: 1
2016-01-14 17:40:16,269 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: D92AEA6F7A9C4306A0CF848338AAF176, mem on start (mb), free: 58, total: 281, max: 672
2016-01-14 17:40:16,269 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,269 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:16,269 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:16,269 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,270 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,270 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:16,270 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:16,271 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:16,271 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:16,271 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:16,271 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:16,272 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, A0935742873842248EF1E3D5DBA6114F, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:16,272 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, A0CAF2DFE79A40B18A23D3DFAAA8C95A, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:16,272 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:16,272 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,272 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: A0CAF2DFE79A40B18A23D3DFAAA8C95A being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:16,278 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:16,278 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:16,284 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:16,286 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:16,290 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: A0CAF2DFE79A40B18A23D3DFAAA8C95A with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:16,292 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:16,293 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:16,297 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:16,302 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,303 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:16,303 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,304 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793214302_0001_1_00_000000_0_10002, runDuration: 392153, ]
2016-01-14 17:40:16,346 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: CoGroup(euclidean)[by:[{1}:'l'][numSelfJoins:1]]
2016-01-14 17:40:16,359 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: CoGroup(euclidean)[by:[{1}:'l'][numSelfJoins:1]] streamed: true, id: 9F815B2E169043A38A913571AA935F90
2016-01-14 17:40:16,361 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(euclidean)[by:[{?}:ALL]], id: FA8824412D004B03B65F05DCA32C5576
2016-01-14 17:40:16,365 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(euclidean)[by:[{?}:ALL]] FA8824412D004B03B65F05DCA32C5576
2016-01-14 17:40:16,368 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793214302_0001_1_00_000000_0_10002] to DISK_DIRECT, CompressedSize=1089, DecompressedSize=1085,EndTime=1452793216368, TimeTaken=9, Rate=0.12 MB/s
2016-01-14 17:40:16,369 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,369 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@5cfcc9dc; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@5828ba35; comparator=cascading.tuple.hadoop.util.TupleComparator@59d91aca; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:16,371 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:16,371 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:16,372 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:16,372 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:16,381 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:16,382 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:16,382 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:16,382 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:16,382 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:16,381 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [A0CAF2DFE79A40B18A23D3DFAAA8C95A] #1 in 24ms
2016-01-14 17:40:16,384 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [A0CAF2DFE79A40B18A23D3DFAAA8C95A] #1, status:false, isInterrupted:false
2016-01-14 17:40:16,387 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:16,388 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: CoGroup(euclidean)[by:[{1}:'l'][numSelfJoins:1]] 9F815B2E169043A38A913571AA935F90, for 1 inputs
2016-01-14 17:40:16,389 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 1089 bytes from disk
2016-01-14 17:40:16,390 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:16,394 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:16,395 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(89)) - attempting to load codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:16,395 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(94)) - found codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:16,399 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 1061 bytes
2016-01-14 17:40:16,399 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,399 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:16,400 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: D92AEA6F7A9C4306A0CF848338AAF176, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:16,404 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@7b0ee504; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@edda8ba
2016-01-14 17:40:16,450 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,450 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: D92AEA6F7A9C4306A0CF848338AAF176, mem on close (mb), free: 89, total: 281, max: 672
2016-01-14 17:40:16,450 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: A0CAF2DFE79A40B18A23D3DFAAA8C95A
2016-01-14 17:40:16,451 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:16,451 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:16,452 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:16,452 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:16,452 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:16,453 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:16,453 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:16,453 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 13104; bufvoid = 66060288
2016-01-14 17:40:16,454 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514376(66057504); length = 693/4128768
2016-01-14 17:40:16,487 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,491 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793214302_0001_1_01_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:16,492 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793214302_0001_1_01, eventCount=1
2016-01-14 17:40:16,493 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:16,495 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=37, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=95, MERGE_PHASE_TIME=113, FIRST_EVENT_RECEIVED=70, LAST_EVENT_RECEIVED=70][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793216401, Process_Duration=49, Process_End_Time=1452793216450, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=11]]
2016-01-14 17:40:16,496 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=D92AEA6F7A9C4306A0CF848338AAF176, taskAttemptId=attempt_1452793214302_0001_1_01_000000_0, startTime=1452793216204, finishTime=1452793216492, timeTaken=288, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=37, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=95, MERGE_PHASE_TIME=113, FIRST_EVENT_RECEIVED=70, LAST_EVENT_RECEIVED=70, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216401, Process_Duration=49, Process_End_Time=1452793216450, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=11
2016-01-14 17:40:16,496 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_01_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:16,496 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16,497 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:16,501 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,500 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:16,502 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000002
2016-01-14 17:40:16,498 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:16,503 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_FINISHED]: vertexName=D92AEA6F7A9C4306A0CF848338AAF176, taskId=task_1452793214302_0001_1_01_000000, startTime=1452793216204, finishTime=1452793216496, timeTaken=292, status=SUCCEEDED, successfulAttemptID=attempt_1452793214302_0001_1_01_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=37, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=95, MERGE_PHASE_TIME=113, FIRST_EVENT_RECEIVED=70, LAST_EVENT_RECEIVED=70, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216401, Process_Duration=49, Process_End_Time=1452793216450, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=11
2016-01-14 17:40:16,504 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_01_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:16,505 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] : 1
2016-01-14 17:40:16,505 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:16,506 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:16,509 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_FINISHED]: vertexName=D92AEA6F7A9C4306A0CF848338AAF176, vertexId=vertex_1452793214302_0001_1_01, initRequestedTime=1452793215390, initedTime=1452793215450, startRequestedTime=1452793215584, startedTime=1452793215584, finishTime=1452793216505, timeTaken=921, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=37, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=295174144, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=95, MERGE_PHASE_TIME=113, FIRST_EVENT_RECEIVED=70, LAST_EVENT_RECEIVED=70, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216401, Process_Duration=49, Process_End_Time=1452793216450, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=11, vertexStats=firstTaskStartTime=1452793216204, firstTasksToStart=[ task_1452793214302_0001_1_01_000000 ], lastTaskFinishTime=1452793216492, lastTasksToFinish=[ task_1452793214302_0001_1_01_000000 ], minTaskDuration=288, maxTaskDuration=288, avgTaskDuration=288.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793214302_0001_1_01_000000 ], longestDurationTasks=[ task_1452793214302_0001_1_01_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:16,510 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:16,510 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000002 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:16,510 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:16,511 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793214302_0001_00_000002, stoppedTime=1452793216510, exitStatus=0
2016-01-14 17:40:16,511 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000002 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:16,511 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] attempt: attempt_1452793214302_0001_1_01_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:16,512 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,512 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:16,512 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793214302_0001_00_000002 is valid, but no longer registered, and will be killed
2016-01-14 17:40:16,513 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000002: 11 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:16,513 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793214302_0001_00_000002, Exiting
2016-01-14 17:40:16,513 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793214302_0001_00_000002
2016-01-14 17:40:16,513 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793214302_0001_00_000002
2016-01-14 17:40:16,514 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793214302_0001_00_000002 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:16,514 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793214302_0001_00_000002 completed successfully
2016-01-14 17:40:16,514 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176] completed., numCompletedVertices=2, numSuccessfulVertices=2, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:16,515 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(242)) - Ignoring stop request for containerId: container_1452793214302_0001_00_000002
2016-01-14 17:40:16,516 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_STARTED]: vertexName=A0935742873842248EF1E3D5DBA6114F, taskId=task_1452793214302_0001_1_02_000000, scheduledTime=1452793216515, launchTime=1452793216515
2016-01-14 17:40:16,517 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:16,518 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_02_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:16,518 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000002 transitioned from STOP_REQUESTED to COMPLETED via event C_COMPLETED
2016-01-14 17:40:16,519 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:16,521 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: A0935742873842248EF1E3D5DBA6114F, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=D92AEA6F7A9C4306A0CF848338AAF176, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A56BB627E00347C6901A0CD066814E35, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:16,521 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_02_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:16,521 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:16,523 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793214302_0001_00_000003
2016-01-14 17:40:16,523 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:16,526 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793214302_0001_00_000003
2016-01-14 17:40:16,527 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:16,529 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:16,529 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793214302_0001_00_000003, launchTime=1452793216529
2016-01-14 17:40:16,529 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:16,531 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000003
2016-01-14 17:40:16,531 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793214302_0001_1_02_000000_0] to container: [container_1452793214302_0001_00_000003]
2016-01-14 17:40:16,531 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:16,532 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793214302_0001_00_000003 given task: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,532 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000003: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,533 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793214302_0001_1_02_000000_0] started. Is using containerId: [container_1452793214302_0001_00_000003] on NM: [127.0.0.1:0]
2016-01-14 17:40:16,533 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:16,533 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=A0935742873842248EF1E3D5DBA6114F, taskAttemptId=attempt_1452793214302_0001_1_02_000000_0, startTime=1452793216532, containerId=container_1452793214302_0001_00_000003, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793214302_0001_00_000003/teamcity, completedLogs=
2016-01-14 17:40:16 Starting to run new task attempt: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,534 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_02_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:16,534 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_02_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:16,534 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,535 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,535 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:16,535 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:16,536 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:16,536 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:16,536 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:16,537 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: A0935742873842248EF1E3D5DBA6114F, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=D92AEA6F7A9C4306A0CF848338AAF176, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A56BB627E00347C6901A0CD066814E35, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:16,537 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:16,538 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,540 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:16,540 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,540 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793214302_0001_1_02_000000_0 sent events: (0-1)
2016-01-14 17:40:16,541 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=D92AEA6F7A9C4306A0CF848338AAF176, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:16,542 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=A56BB627E00347C6901A0CD066814E35, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:16,541 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,542 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:16,542 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:16,543 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,543 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: A56BB627E00347C6901A0CD066814E35
2016-01-14 17:40:16,545 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:16,546 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:16,546 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:16,546 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: A56BB627E00347C6901A0CD066814E35
2016-01-14 17:40:16,546 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: A56BB627E00347C6901A0CD066814E35
2016-01-14 17:40:16,546 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,547 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:16,548 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,569 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: A0935742873842248EF1E3D5DBA6114F, ordinal: 2
2016-01-14 17:40:16,569 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A0935742873842248EF1E3D5DBA6114F, mem on start (mb), free: 82, total: 281, max: 672
2016-01-14 17:40:16,569 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,569 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:16,569 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:16,569 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:16,569 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:16,570 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:16,570 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:16,570 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:16,570 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:16,570 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, A56BB627E00347C6901A0CD066814E35, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:16,571 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, D92AEA6F7A9C4306A0CF848338AAF176, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:16,571 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:16,571 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: D92AEA6F7A9C4306A0CF848338AAF176 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:16,571 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,571 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:16,574 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:16,575 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:16,577 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:16,577 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: D92AEA6F7A9C4306A0CF848338AAF176 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:16,577 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:16,577 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:16,578 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:16,581 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,581 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:16,582 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793214302_0001_1_01_000000_0_10006, runDuration: 253453, ]
2016-01-14 17:40:16,582 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,585 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(euclidean)[by:[{?}:ALL]]
2016-01-14 17:40:16,585 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793214302_0001_1_01_000000_0_10006] to DISK_DIRECT, CompressedSize=7519, DecompressedSize=7515,EndTime=1452793216585, TimeTaken=1, Rate=7.17 MB/s
2016-01-14 17:40:16,585 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,586 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.01 MB/s)
2016-01-14 17:40:16,586 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [D92AEA6F7A9C4306A0CF848338AAF176] #2 in 2ms
2016-01-14 17:40:16,586 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [D92AEA6F7A9C4306A0CF848338AAF176] #2, status:false, isInterrupted:false
2016-01-14 17:40:16,587 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:16,587 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 7519 bytes from disk
2016-01-14 17:40:16,587 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:16,587 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:16,587 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(euclidean)[by:[{?}:ALL]] streamed: true, id: FA8824412D004B03B65F05DCA32C5576
2016-01-14 17:40:16,588 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(euclidean)[by:[{2}:'n1', 'n2']], id: 53823E81C36D46BE9DF05C680BFC9BC8
2016-01-14 17:40:16,588 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 7443 bytes
2016-01-14 17:40:16,588 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(euclidean)[by:[{2}:'n1', 'n2']] 53823E81C36D46BE9DF05C680BFC9BC8
2016-01-14 17:40:16,588 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,589 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:16,591 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@73f2a23c; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@3e794c9c; comparator=cascading.tuple.hadoop.util.TupleComparator@d94b5cf; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:16,591 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:16,591 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:16,592 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:16,663 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:16,663 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:16,663 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:16,663 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:16,664 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:16,664 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(euclidean)[by:[{?}:ALL]] FA8824412D004B03B65F05DCA32C5576, for 1 inputs
2016-01-14 17:40:16,665 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: A0935742873842248EF1E3D5DBA6114F, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:16,665 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@795aa654; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@2e5e130a
2016-01-14 17:40:16,694 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,694 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A0935742873842248EF1E3D5DBA6114F, mem on close (mb), free: 78, total: 345, max: 672
2016-01-14 17:40:16,694 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: D92AEA6F7A9C4306A0CF848338AAF176
2016-01-14 17:40:16,694 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:16,695 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:16,695 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:16,695 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:16,695 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:16,695 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:16,695 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:16,696 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 6291; bufvoid = 66060288
2016-01-14 17:40:16,696 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514724(66058896); length = 345/4128768
2016-01-14 17:40:16,702 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,706 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793214302_0001_1_02_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:16,707 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793214302_0001_1_02, eventCount=1
2016-01-14 17:40:16,708 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:16,708 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=9, MERGE_PHASE_TIME=11, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793216665, Process_Duration=28, Process_End_Time=1452793216693, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=8]]
2016-01-14 17:40:16,709 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:16,711 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,709 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=A0935742873842248EF1E3D5DBA6114F, taskAttemptId=attempt_1452793214302_0001_1_02_000000_0, startTime=1452793216532, finishTime=1452793216707, timeTaken=175, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=9, MERGE_PHASE_TIME=11, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216665, Process_Duration=28, Process_End_Time=1452793216693, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=8
2016-01-14 17:40:16,713 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_02_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:16,714 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16,714 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:16,713 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000003
2016-01-14 17:40:16,716 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:16,718 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_FINISHED]: vertexName=A0935742873842248EF1E3D5DBA6114F, taskId=task_1452793214302_0001_1_02_000000, startTime=1452793216532, finishTime=1452793216714, timeTaken=182, status=SUCCEEDED, successfulAttemptID=attempt_1452793214302_0001_1_02_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=9, MERGE_PHASE_TIME=11, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216665, Process_Duration=28, Process_End_Time=1452793216693, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=8
2016-01-14 17:40:16,718 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_02_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:16,719 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] : 1
2016-01-14 17:40:16,719 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:16,720 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:16,722 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_FINISHED]: vertexName=A0935742873842248EF1E3D5DBA6114F, vertexId=vertex_1452793214302_0001_1_02, initRequestedTime=1452793215456, initedTime=1452793215482, startRequestedTime=1452793215595, startedTime=1452793215595, finishTime=1452793216719, timeTaken=1124, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=9, MERGE_PHASE_TIME=11, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216665, Process_Duration=28, Process_End_Time=1452793216693, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=8, vertexStats=firstTaskStartTime=1452793216532, firstTasksToStart=[ task_1452793214302_0001_1_02_000000 ], lastTaskFinishTime=1452793216707, lastTasksToFinish=[ task_1452793214302_0001_1_02_000000 ], minTaskDuration=175, maxTaskDuration=175, avgTaskDuration=175.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793214302_0001_1_02_000000 ], longestDurationTasks=[ task_1452793214302_0001_1_02_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:16,723 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:16,723 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:16,723 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:16,723 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793214302_0001_00_000003, stoppedTime=1452793216723, exitStatus=0
2016-01-14 17:40:16,724 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:16,725 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] attempt: attempt_1452793214302_0001_1_02_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:16,725 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: A56BB627E00347C6901A0CD066814E35
2016-01-14 17:40:16,725 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:16,726 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F] completed., numCompletedVertices=3, numSuccessfulVertices=3, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:16,726 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793214302_0001_00_000003. Relying on regular task shutdown for it to end
2016-01-14 17:40:16,726 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793214302_0001_00_000003 is valid, but no longer registered, and will be killed
2016-01-14 17:40:16,727 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000003: 11 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:16,727 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793214302_0001_00_000003, Exiting
2016-01-14 17:40:16,727 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793214302_0001_00_000003
2016-01-14 17:40:16,727 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793214302_0001_00_000003
2016-01-14 17:40:16,728 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:16,728 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793214302_0001_00_000003 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:16,728 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793214302_0001_00_000003 completed successfully
2016-01-14 17:40:16,728 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_STARTED]: vertexName=A56BB627E00347C6901A0CD066814E35, taskId=task_1452793214302_0001_1_03_000000, scheduledTime=1452793216727, launchTime=1452793216727
2016-01-14 17:40:16,729 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_03_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:16,729 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:16,730 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:16,730 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000003 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:16,731 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: A56BB627E00347C6901A0CD066814E35, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=A0935742873842248EF1E3D5DBA6114F, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=069C4602D28844528CDE09F3C716F571, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:16,731 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_03_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:16,731 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:16,733 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793214302_0001_00_000004
2016-01-14 17:40:16,733 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000004 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:16,735 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793214302_0001_00_000004
2016-01-14 17:40:16,736 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:16,738 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:16,738 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793214302_0001_00_000004, launchTime=1452793216738
2016-01-14 17:40:16,738 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000004 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:16,739 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000004
2016-01-14 17:40:16,740 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793214302_0001_1_03_000000_0] to container: [container_1452793214302_0001_00_000004]
2016-01-14 17:40:16,740 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000004 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:16,740 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793214302_0001_00_000004 given task: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,740 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000004: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,741 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793214302_0001_1_03_000000_0] started. Is using containerId: [container_1452793214302_0001_00_000004] on NM: [127.0.0.1:0]
2016-01-14 17:40:16,741 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:16,741 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=A56BB627E00347C6901A0CD066814E35, taskAttemptId=attempt_1452793214302_0001_1_03_000000_0, startTime=1452793216740, containerId=container_1452793214302_0001_00_000004, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793214302_0001_00_000004/teamcity, completedLogs=
2016-01-14 17:40:16,741 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_03_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:16,742 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_03_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:16 Starting to run new task attempt: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,742 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,742 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,742 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:16,742 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:16,743 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:16,743 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:16,743 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:16,743 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7] euclidean/(1/1) ...osite/euclidean/composite, VertexName: A56BB627E00347C6901A0CD066814E35, VertexParallelism: 1, TaskAttemptID:attempt_1452793214302_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=A0935742873842248EF1E3D5DBA6114F, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=069C4602D28844528CDE09F3C716F571, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:16,743 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:16,745 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793214302_0001_1_03_000000_0 sent events: (0-1)
2016-01-14 17:40:16,746 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,746 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:16,747 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,748 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=A0935742873842248EF1E3D5DBA6114F, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:16,748 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=069C4602D28844528CDE09F3C716F571, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }
2016-01-14 17:40:16,748 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,749 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:16,749 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,751 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:16,752 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:16,752 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,753 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,753 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:16,755 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:16,756 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: 069C4602D28844528CDE09F3C716F571
2016-01-14 17:40:16,756 INFO  output.MROutput (MROutput.java:initialize(347)) - Initializing Simple Output
2016-01-14 17:40:16,756 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: OUTPUT, componentVertexName: 069C4602D28844528CDE09F3C716F571
2016-01-14 17:40:16,777 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: A56BB627E00347C6901A0CD066814E35, ordinal: 3
2016-01-14 17:40:16,777 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A56BB627E00347C6901A0CD066814E35, mem on start (mb), free: 72, total: 345, max: 672
2016-01-14 17:40:16,778 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:16,778 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:16,778 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:16,784 INFO  output.MROutput (MROutput.java:initialize(418)) - Initialized Simple Output, using_new_api: false
2016-01-14 17:40:16,784 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: 069C4602D28844528CDE09F3C716F571
2016-01-14 17:40:16,785 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:16,785 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:16,786 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:16,786 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.output.MROutput
2016-01-14 17:40:16,787 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:16,787 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:16,787 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 634178752, TotalRequestedScaled: 5.853957710769231E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.90
2016-01-14 17:40:16,788 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, A0935742873842248EF1E3D5DBA6114F, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=472110858
2016-01-14 17:40:16,788 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, 069C4602D28844528CDE09F3C716F571, org.apache.tez.mapreduce.output.MROutput: requested=0, allocated=0
2016-01-14 17:40:16,788 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:16,788 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: A0935742873842248EF1E3D5DBA6114F being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:16,788 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:16,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:16,790 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:16,790 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:16,790 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: A0935742873842248EF1E3D5DBA6114F with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:16,791 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=472110858. Updated to: ShuffleMem=472110858, postMergeMem: 0
2016-01-14 17:40:16,791 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=472110858, maxSingleShuffleLimit=118027712, mergeThreshold=424899744, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:16,791 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:16,792 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,793 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:16,793 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,797 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(euclidean)[by:[{2}:'n1', 'n2']]
2016-01-14 17:40:16,797 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793214302_0001_1_02_000000_0_10009, runDuration: 159629, ]
2016-01-14 17:40:16,799 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(euclidean)[by:[{2}:'n1', 'n2']] streamed: true, id: 53823E81C36D46BE9DF05C680BFC9BC8
2016-01-14 17:40:16,800 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceComposite/euclidean/composite"], id: 069C4602D28844528CDE09F3C716F571
2016-01-14 17:40:16,802 INFO  element.TezSinkStage (TezSinkStage.java:prepare(57)) - calling MROutput#start() on: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceComposite/euclidean/composite"]
2016-01-14 17:40:16,803 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(euclidean)[by:[{2}:'n1', 'n2']] 53823E81C36D46BE9DF05C680BFC9BC8, for 1 inputs
2016-01-14 17:40:16,804 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793214302_0001_1_02_000000_0_10009] to DISK_DIRECT, CompressedSize=4344, DecompressedSize=4340,EndTime=1452793216804, TimeTaken=4, Rate=1.04 MB/s
2016-01-14 17:40:16,804 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,805 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:16,805 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [A0935742873842248EF1E3D5DBA6114F] #3 in 5ms
2016-01-14 17:40:16,805 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [A0935742873842248EF1E3D5DBA6114F] #3, status:false, isInterrupted:false
2016-01-14 17:40:16,806 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:16,806 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 4344 bytes from disk
2016-01-14 17:40:16,806 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:16,807 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:16,807 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 4301 bytes
2016-01-14 17:40:16,807 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,807 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:16,808 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: A56BB627E00347C6901A0CD066814E35, all 1 inputs ready in: 00:00:00.004
2016-01-14 17:40:16,808 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@2052c9cf; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@1b3f459d
2016-01-14 17:40:16,821 INFO  output.MROutput (MROutput.java:flush(534)) - Flushing Simple Output
2016-01-14 17:40:16,822 INFO  output.MROutput (MROutput.java:flush(544)) - Flushed Simple Output
2016-01-14 17:40:16,823 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:canCommit(353)) - Commit go/no-go request from attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,824 INFO  impl.TaskImpl (TaskImpl.java:canCommit(750)) - attempt_1452793214302_0001_1_03_000000_0 given a go for committing the task output.
2016-01-14 17:40:16,825 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,826 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A56BB627E00347C6901A0CD066814E35, mem on close (mb), free: 69, total: 345, max: 672
2016-01-14 17:40:16,826 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: A0935742873842248EF1E3D5DBA6114F
2016-01-14 17:40:16,826 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:16,827 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:16,828 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:16,828 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:16,828 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,832 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793214302_0001_1_03_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:16,833 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:16,835 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=A56BB627E00347C6901A0CD066814E35, taskAttemptId=attempt_1452793214302_0001_1_03_000000_0, startTime=1452793216740, finishTime=1452793216833, timeTaken=93, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=15, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216808, Process_Duration=17, Process_End_Time=1452793216825, Read_Duration=7, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:16,835 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793214302_0001_1_03_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:16,835 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:16,835 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16,836 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:16,837 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:TASK_FINISHED]: vertexName=A56BB627E00347C6901A0CD066814E35, taskId=task_1452793214302_0001_1_03_000000, startTime=1452793216740, finishTime=1452793216836, timeTaken=96, status=SUCCEEDED, successfulAttemptID=attempt_1452793214302_0001_1_03_000000_0, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=15, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216808, Process_Duration=17, Process_End_Time=1452793216825, Read_Duration=7, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:16,837 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 46 [[File System Counters FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=15, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793216808, Process_Duration=17, Process_End_Time=1452793216825, Read_Duration=7, Tuples_Read=87, Tuples_Written=21, Write_Duration=0][cascading.flow.StepCounters Tuples_Written=21]]
2016-01-14 17:40:16,837 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:16,838 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,838 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793214302_0001_00_000004
2016-01-14 17:40:16,839 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793214302_0001_1_03_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:16,840 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] : 1
2016-01-14 17:40:16,840 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:16,841 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:16,842 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:VERTEX_FINISHED]: vertexName=A56BB627E00347C6901A0CD066814E35, vertexId=vertex_1452793214302_0001_1_03, initRequestedTime=1452793215506, initedTime=1452793215557, startRequestedTime=1452793215597, startedTime=1452793215597, finishTime=1452793216840, timeTaken=1243, status=SUCCEEDED, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=361758720, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=15, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793216808, Process_Duration=17, Process_End_Time=1452793216825, Read_Duration=7, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21, vertexStats=firstTaskStartTime=1452793216740, firstTasksToStart=[ task_1452793214302_0001_1_03_000000 ], lastTaskFinishTime=1452793216833, lastTasksToFinish=[ task_1452793214302_0001_1_03_000000 ], minTaskDuration=93, maxTaskDuration=93, avgTaskDuration=93.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793214302_0001_1_03_000000 ], longestDurationTasks=[ task_1452793214302_0001_1_03_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:16,845 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:16,845 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000004 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:16,846 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793214302_0001_00_000004, stoppedTime=1452793216846, exitStatus=0
2016-01-14 17:40:16,846 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000004 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:16,846 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:16,847 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35] completed., numCompletedVertices=4, numSuccessfulVertices=4, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:16,847 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(831)) - Calling DAG commit/abort for dag: dag_1452793214302_0001_1
2016-01-14 17:40:16,848 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:DAG_COMMIT_STARTED]: dagID=dag_1452793214302_0001_1
2016-01-14 17:40:16,849 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793214302_0001_1_02 [A0935742873842248EF1E3D5DBA6114F]
2016-01-14 17:40:16,849 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793214302_0001_1_01 [D92AEA6F7A9C4306A0CF848338AAF176]
2016-01-14 17:40:16,849 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(892)) - Committing output: 069C4602D28844528CDE09F3C716F571 for vertex: vertex_1452793214302_0001_1_03 [A56BB627E00347C6901A0CD066814E35]
2016-01-14 17:40:16,849 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793214302_0001_00_000004 is valid, but no longer registered, and will be killed
2016-01-14 17:40:16,851 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793214302_0001_00_000004: 12 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:16,851 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793214302_0001_00_000004, Exiting
2016-01-14 17:40:16,851 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793214302_0001_00_000004
2016-01-14 17:40:16,851 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793214302_0001_00_000004
2016-01-14 17:40:16,854 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793214302_0001_00_000004 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:16,854 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793214302_0001_00_000004 completed successfully
2016-01-14 17:40:16,865 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793214302_0001_1_00 [A0CAF2DFE79A40B18A23D3DFAAA8C95A]
2016-01-14 17:40:16,874 INFO  recovery.RecoveryService (RecoveryService.java:handle(306)) - DAG completed, dagId=dag_1452793214302_0001_1, queueSize=0
2016-01-14 17:40:16,874 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_FINISHED to history file
2016-01-14 17:40:16,875 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793214302_0001_1][Event:DAG_FINISHED]: dagId=dag_1452793214302_0001_1, startTime=1452793215342, finishTime=1452793216865, timeTaken=1523, status=SUCCEEDED, diagnostics=, counters=Counters: 55, org.apache.tez.common.counters.DAGCounter, NUM_SUCCEEDED_TASKS=4, TOTAL_LAUNCHED_TASKS=4, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=14386, FILE_BYTES_WRITTEN=13948, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=111, REDUCE_INPUT_RECORDS=296, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=592, NUM_SHUFFLED_INPUTS=3, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=3, GC_TIME_MILLIS=37, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=1313865728, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=317, OUTPUT_BYTES=21071, OUTPUT_BYTES_WITH_OVERHEAD=12940, OUTPUT_BYTES_PHYSICAL=12952, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=12952, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=12952, SHUFFLE_BYTES_DECOMPRESSED=12940, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=12952, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=119, MERGE_PHASE_TIME=141, FIRST_EVENT_RECEIVED=86, LAST_EVENT_RECEIVED=86, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=5811172865954, Process_Duration=104, Process_End_Time=5811172866058, Read_Duration=20, Tuples_Read=303, Tuples_Written=317, Write_Duration=21, cascading.flow.StepCounters, Tuples_Read=7, Tuples_Written=21
2016-01-14 17:40:16,876 INFO  impl.DAGImpl (DAGImpl.java:finished(1180)) - DAG: dag_1452793214302_0001_1 finished with state: SUCCEEDED
2016-01-14 17:40:16,876 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793214302_0001_1 transitioned from RUNNING to SUCCEEDED due to event DAG_VERTEX_COMPLETED
2016-01-14 17:40:16,877 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(242)) - Ignoring stop request for containerId: container_1452793214302_0001_00_000004
2016-01-14 17:40:16,878 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793214302_0001_00_000004 transitioned from STOP_REQUESTED to COMPLETED via event C_COMPLETED
2016-01-14 17:40:16 Completed Dag: dag_1452793214302_0001_1
2016-01-14 17:40:16,879 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(579)) - DAG completed, dagId=dag_1452793214302_0001_1, dagState=SUCCEEDED
2016-01-14 17:40:16,880 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793214302_0001_1_post
2016-01-14 17:40:16,880 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:16,880 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(625)) - Waiting for next DAG to be submitted.
2016-01-14 17:40:16,889 INFO  client.TezClient (TezClient.java:stop(495)) - Shutting down Tez Session, sessionName=(1/1) ...osite/euclidean/composite, applicationId=application_1452793214302_0001
2016-01-14 17:40:16,893 INFO  client.DAGClientHandler (DAGClientHandler.java:shutdownAM(117)) - Received message to shutdown AM
2016-01-14 17:40:16,893 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:setShouldUnregisterFlag(580)) - TaskScheduler notified that it should unregister from RM
2016-01-14 17:40:16,894 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdownTezAM(1088)) - No current running DAG, shutting down the AM
2016-01-14 17:40:16,894 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(692)) - DAGAppMasterShutdownHandler invoked
2016-01-14 17:40:16,894 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(701)) - Handling DAGAppMaster shutdown
2016-01-14 17:40:16,895 INFO  app.DAGAppMaster (DAGAppMaster.java:run(721)) - Sleeping for 5 seconds before shutting down
2016-01-14 17:40:16,897 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStop(79)) - Stopping HistoryEventHandler
2016-01-14 17:40:16,897 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(189)) - Stopping RecoveryService
2016-01-14 17:40:16,897 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(192)) - Handle the remaining events in queue, queue size=0
2016-01-14 17:40:16,898 INFO  recovery.RecoveryService (RecoveryService.java:run(166)) - EventQueue take interrupted. Returning
2016-01-14 17:40:16,898 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(214)) - Closing Summary Stream
2016-01-14 17:40:16,898 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStop(118)) - Stopping SimpleHistoryLoggingService, eventQueueBacklog=0
2016-01-14 17:40:16,898 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:run(105)) - EventQueue take interrupted. Returning
2016-01-14 17:40:16,905 INFO  app.DAGAppMaster (DAGAppMaster.java:run(1773)) - Completed deletion of tez scratch data dir, path=file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/D130B26C403F43AB8A8535427D5AA2CB/DBBD0DA22DD14961A6D3811DD4B9D3C7/.tez/application_1452793214302_0001
2016-01-14 17:40:16,906 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  completed in: 00:02.791
2016-01-14 17:40:17,043 INFO  cascading.PlatformTestCase (PlatformTestCase.java:tearDown(189)) - copying to local /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output//hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceComposite/euclidean/composite
2016-01-14 17:40:17,086 INFO  util.Util (Util.java:findMainClass(1327)) - using default application jar, may cause class not found exceptions on the cluster
2016-01-14 17:40:17,086 INFO  planner.Hadoop2TezPlanner (Hadoop2TezPlanner.java:initialize(149)) - using application jar: /opt/concurrent/home/teamcity/.gradle/caches/modules-2/files-2.1/cascading/cascading-hadoop2-tez/3.0.4/561447ee876073bb61fddc52e2ad157f45c8edd/cascading-hadoop2-tez-3.0.4.jar
2016-01-14 17:40:17,178 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] executed rule registry: NoHashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.056
2016-01-14 17:40:17,217 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] executed rule registry: HashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.095
2016-01-14 17:40:17,218 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] rule registry: NoHashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:17,218 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] rule registry: HashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:17,218 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] rule registry: NoHashJoinHadoop2TezRuleRegistry, result was selected using: 'default comparator: selects plan with fewest steps and fewest nodes'
2016-01-14 17:40:17,245 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt
2016-01-14 17:40:17,279 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testPearsonDistanceComposite/pearson/composite
2016-01-14 17:40:17,457 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] starting
2016-01-14 17:40:17,457 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson]  source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:17,457 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson]  sink: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testPearsonDistanceComposite/pearson/composite"]
2016-01-14 17:40:17,458 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson]  parallel execution of steps is enabled: false
2016-01-14 17:40:17,458 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson]  executing total steps: 1
2016-01-14 17:40:17,458 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson]  allocating management threads: 1
2016-01-14 17:40:17,459 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] starting step: (1/1) ...mposite/pearson/composite
2016-01-14 17:40:17,459 WARN  flow.Flow (BaseFlow.java:logWarn(1445)) - [pearson] 'yarn.timeline-service.enabled' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon
2016-01-14 17:40:17,476 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] tez session mode enabled: true
2016-01-14 17:40:17,481 INFO  client.TezClient (TezClient.java:<init>(153)) - Tez Client Version: [ component=tez-api, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:17,481 INFO  client.TezClient (TezClient.java:start(333)) - Session mode. Starting session.
2016-01-14 17:40:17,481 INFO  client.TezClientUtils (TezClientUtils.java:setupTezJarsLocalResources(159)) - Ignoring 'tez.lib.uris' since  'tez.ignore.lib.uris' is set to true
2016-01-14 17:40:17,491 INFO  client.TezClient (TezCommonUtils.java:createTezSystemStagingPath(118)) - Tez system stage directory file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001 doesn't exist and is created
2016-01-14 17:40:17,507 INFO  client.LocalClient (LocalClient.java:createDAGAppMaster(328)) - DAGAppMaster thread has been created
2016-01-14 17:40:17,507 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:17,508 INFO  client.LocalClient (LocalClient.java:run(275)) - Using staging directory: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001
2016-01-14 17:40:17,511 INFO  app.DAGAppMaster (DAGAppMaster.java:<init>(309)) - Created DAGAppMaster for application appattempt_1452793217481_0001_000000, versionInfo=[ component=tez-dag, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:17,525 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(333)) - Comparing client version with AM version, clientVersion=0.6.2, AMVersion=0.6.2
2016-01-14 17:40:17,527 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(386)) - Adding session token to jobTokenSecretManager for application
2016-01-14 17:40:17,529 INFO  node.AMNodeTracker (AMNodeTracker.java:serviceInit(77)) - blacklistDisablePercent is 33, blacklistingEnabled: false, maxTaskFailuresPerNode: 1
2016-01-14 17:40:17,529 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceInit(50)) - Initializing HistoryEventHandler
2016-01-14 17:40:17,529 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(66)) - Log file location for SimpleHistoryLoggingService not specified, defaulting to containerLogDir=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001/localmode-log-dir
2016-01-14 17:40:17,530 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(88)) - Initializing SimpleHistoryLoggingService, logFileLocation=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001/localmode-log-dir/history.txt.appattempt_1452793217481_0001_000000, maxErrors=10
2016-01-14 17:40:17,530 INFO  recovery.RecoveryService (RecoveryService.java:serviceInit(109)) - Initializing RecoveryService
2016-01-14 17:40:17,530 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_LAUNCHED]: appAttemptId=appattempt_1452793217481_0001_000000, appSubmitTime=1452793217510, launchTime=1452793217510
2016-01-14 17:40:17,538 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStart(73)) - Starting HistoryEventHandler
2016-01-14 17:40:17,539 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStart(95)) - Starting SimpleHistoryLoggingService
2016-01-14 17:40:17,541 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_LAUNCHED to history file
2016-01-14 17:40:17,543 INFO  recovery.RecoveryService (RecoveryService.java:serviceStart(127)) - Starting RecoveryService
2016-01-14 17:40:17,548 INFO  client.DAGClientServer (DAGClientServer.java:serviceStart(91)) - Instantiated DAGClientRPCServer at ip-10-187-41-40.ec2.internal/10.187.41.40:56752
2016-01-14 17:40:17,550 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-dagappmaster.properties,hadoop-metrics2.properties
2016-01-14 17:40:17,552 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_STARTED]: appAttemptId=appattempt_1452793217481_0001_000000, startTime=1452793217552
2016-01-14 17:40:17,552 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_STARTED to history file
2016-01-14 17:40:17,552 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceStart(1644)) - In Session mode. Waiting for DAG over RPC
2016-01-14 17:40:17,607 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: IDLE
2016-01-14 17:40:17,608 INFO  client.TezClient (TezClient.java:start(367)) - The url to track the Tez Session: N/A
2016-01-14 17:40:17,608 INFO  client.TezClient (TezClient.java:submitDAGSession(405)) - Submitting dag to TezSession, sessionName=(1/1) ...mposite/pearson/composite, applicationId=application_1452793217481_0001, dagName=[A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite
2016-01-14 17:40:17,631 INFO  app.DAGAppMaster (DAGAppMaster.java:submitDAGToAppMaster(1126)) - Starting DAG submitted via RPC: [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite
2016-01-14 17:40:17,634 INFO  app.DAGAppMaster (DAGAppMaster.java:generateDAGVizFile(870)) - Generating DAG graphviz file, dagId=dag_1452793217481_0001_1, filePath=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001/localmode-log-dir/dag_1452793217481_0001_1.dot
2016-01-14 17:40:17,637 INFO  app.DAGAppMaster (DAGAppMaster.java:writePBTextFile(887)) - Writing DAG plan to: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001/localmode-log-dir/dag_1452793217481_0001_1-tez-dag.pb.txt
2016-01-14 17:40:17,675 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793217481_0001_1
2016-01-14 17:40:17,675 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:17,675 INFO  app.DAGAppMaster (DAGAppMaster.java:startDAG(2051)) - Running DAG: [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite
2016-01-14 17:40:17 Running Dag: dag_1452793217481_0001_1
2016-01-14 17:40:17,689 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:DAG_SUBMITTED]: dagID=dag_1452793217481_0001_1, submitTime=1452793217631
2016-01-14 17:40:17,689 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_SUBMITTED to history file
2016-01-14 17:40:17,690 INFO  impl.VertexImpl (VertexImpl.java:setAdditionalOutputs(4033)) - setting additional outputs for vertex C56ED17F7195422F8D2061A379608738
2016-01-14 17:40:17,691 INFO  impl.DAGImpl (DAGImpl.java:assignDAGScheduler(1381)) - Using DAG Scheduler: org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
2016-01-14 17:40:17,691 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:DAG_INITIALIZED]: dagID=dag_1452793217481_0001_1, initTime=1452793217689
2016-01-14 17:40:17,693 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_INITIALIZED to history file
2016-01-14 17:40:17,693 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793217481_0001_1 transitioned from NEW to INITED due to event DAG_INIT
2016-01-14 17:40:17,694 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:DAG_STARTED]: dagID=dag_1452793217481_0001_1, startTime=1452793217694
2016-01-14 17:40:17,694 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_STARTED to history file
2016-01-14 17:40:17,694 INFO  impl.DAGImpl (DAGImpl.java:transition(1614)) - Added additional resources : [[]] to classpath
2016-01-14 17:40:17,695 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793217481_0001_1 transitioned from INITED to RUNNING due to event DAG_START
2016-01-14 17:40:17,695 INFO  client.TezClient (TezClient.java:submitDAGSession(478)) - Submitted dag to TezSession, sessionName=(1/1) ...mposite/pearson/composite, applicationId=application_1452793217481_0001, dagName=[A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite
2016-01-14 17:40:17,695 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2068)) - Root Inputs exist for Vertex: D67FD865B49048F3B8F908CFF8170DDB : {EC65373C8D9146E3B5B97F9F84B9518E={InputName=EC65373C8D9146E3B5B97F9F84B9518E}, {Descriptor=ClassName=org.apache.tez.mapreduce.input.MRInput, hasPayload=true}, {ControllerDescriptor=ClassName=org.apache.tez.mapreduce.common.MRInputAMSplitGenerator, hasPayload=false}}
2016-01-14 17:40:17,695 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2078)) - Starting root input initializer for input: EC65373C8D9146E3B5B97F9F84B9518E, with class: [org.apache.tez.mapreduce.common.MRInputAMSplitGenerator]
2016-01-14 17:40:17,695 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2191)) - Setting vertexManager to RootInputVertexManager for vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,695 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2983)) - Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split to set #tasks for the vertex vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,696 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2987)) - Vertex will initialize from input initializer. vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,696 INFO  impl.VertexImpl (VertexImpl.java:setupInputInitializerManager(3933)) - Vertex will initialize via inputInitializers vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]. Starting root input initializers: 1
2016-01-14 17:40:17,696 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] transitioned from NEW to INITIALIZING due to event V_INIT
2016-01-14 17:40:17,697 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:17,697 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:run(242)) - Starting InputInitializer for Input: EC65373C8D9146E3B5B97F9F84B9518E on vertex vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,701 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(92)) - Emitting serialized splits: true
2016-01-14 17:40:17,702 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:17,703 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(110)) - Input EC65373C8D9146E3B5B97F9F84B9518E asking for 1 tasks. Headroom: 672 Task Resource: 1024 waves: 1.7
2016-01-14 17:40:17,703 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:17,707 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(121)) - Grouping input splits
2016-01-14 17:40:17,710 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(294)) - Generating mapred api input splits
2016-01-14 17:40:17,713 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(300)) - NumSplits: 1, SerializedSize: 404
2016-01-14 17:40:17,710 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:17,714 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:17,714 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_INITIALIZED]: vertexName=E4BB088620B54C1DA2F37BBDF51DABC7, vertexId=vertex_1452793217481_0001_1_01, initRequestedTime=1452793217697, initedTime=1452793217714, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:17,715 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:17,715 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:17,722 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:17,724 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:17,725 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:17,725 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_INITIALIZED]: vertexName=FAE62F9FD3884070A14642DD2A2C76A9, vertexId=vertex_1452793217481_0001_1_02, initRequestedTime=1452793217715, initedTime=1452793217725, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:17,726 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:17,726 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:17,723 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:onSuccess(272)) - Succeeded InputInitializer for Input: EC65373C8D9146E3B5B97F9F84B9518E on vertex vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,725 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:17,724 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson] submitted tez dag to app master: application_1452793217481_0001, with dag id: dag_1452793217481_0001_1
2016-01-14 17:40:17,735 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:17,735 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:17,737 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:17,737 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1915)) - Invoking committer inits for vertex, vertexId=vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:17,738 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1927)) - Instantiating committer for output=1D79E0D7E49D4EA999A42FBB82B8CDDD, vertexId=vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738], committerClass=org.apache.tez.mapreduce.committer.MROutputCommitter
2016-01-14 17:40:17,739 INFO  impl.VertexImpl (VertexImpl.java:run(1945)) - Invoking committer init for output=1D79E0D7E49D4EA999A42FBB82B8CDDD, vertexId=vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:17,743 INFO  committer.MROutputCommitter (MROutputCommitter.java:getOutputCommitter(145)) - OutputCommitter for outputName=1D79E0D7E49D4EA999A42FBB82B8CDDD, vertexName=C56ED17F7195422F8D2061A379608738, outputCommitterClass=org.apache.hadoop.mapred.FileOutputCommitter
2016-01-14 17:40:17,743 INFO  impl.VertexImpl (VertexImpl.java:run(1949)) - Invoking committer setup for output=1D79E0D7E49D4EA999A42FBB82B8CDDD, vertexId=vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:17,745 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_INITIALIZED]: vertexName=C56ED17F7195422F8D2061A379608738, vertexId=vertex_1452793217481_0001_1_03, initRequestedTime=1452793217726, initedTime=1452793217744, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:17,745 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:17,745 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:17,746 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] numTasks: -1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:17,746 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1374)) - Got updated RootInputsSpecs: {EC65373C8D9146E3B5B97F9F84B9518E=forAllWorkUnits=true, update=[1]}
2016-01-14 17:40:17,747 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1393)) - Vertex vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] parallelism set to 1
2016-01-14 17:40:17,747 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] numTasks: 1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:17,749 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793217481_0001_1_00, eventCount=1
2016-01-14 17:40:17,750 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:17,751 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_INITIALIZED]: vertexName=D67FD865B49048F3B8F908CFF8170DDB, vertexId=vertex_1452793217481_0001_1_00, initRequestedTime=1452793217695, initedTime=1452793217749, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=1
2016-01-14 17:40:17,751 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] with distanceFromRoot: 0
2016-01-14 17:40:17,752 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] transitioned from INITIALIZING to INITED due to event V_ROOT_INPUT_INITIALIZED
2016-01-14 17:40:17,753 INFO  impl.ImmediateStartVertexManager (ImmediateStartVertexManager.java:scheduleTasks(99)) - Starting 1 in D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,753 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,754 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:17,754 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793217481_0001_1_00, startRequestedTime=1452793217753, startedTime=1452793217753
2016-01-14 17:40:17,755 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:17,756 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:17,757 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_STARTED]: vertexName=D67FD865B49048F3B8F908CFF8170DDB, taskId=task_1452793217481_0001_1_00_000000, scheduledTime=1452793217756, launchTime=1452793217756
2016-01-14 17:40:17,757 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_00_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:17,758 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793217481_0001_1_00 for vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] numStartedSources: 1 numSources: 1
2016-01-14 17:40:17,758 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] with distanceFromRoot: 1
2016-01-14 17:40:17,758 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,759 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: D67FD865B49048F3B8F908CFF8170DDB in vertex: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:17,759 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: E4BB088620B54C1DA2F37BBDF51DABC7 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:17,760 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:17,760 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793217481_0001_1_01, startRequestedTime=1452793217759, startedTime=1452793217759
2016-01-14 17:40:17,761 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:17,761 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: D67FD865B49048F3B8F908CFF8170DDB, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=EC65373C8D9146E3B5B97F9F84B9518E, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=E4BB088620B54C1DA2F37BBDF51DABC7, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:17,762 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_00_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:17,762 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793217481_0001_1_01 for vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] numStartedSources: 1 numSources: 1
2016-01-14 17:40:17,762 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] with distanceFromRoot: 2
2016-01-14 17:40:17,763 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:17,764 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: E4BB088620B54C1DA2F37BBDF51DABC7 in vertex: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:17,764 INFO  node.AMNodeTracker (AMNodeTracker.java:nodeSeen(91)) - Adding new node: 127.0.0.1:0
2016-01-14 17:40:17,764 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: FAE62F9FD3884070A14642DD2A2C76A9 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:17,765 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:17,765 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793217481_0001_1_02, startRequestedTime=1452793217763, startedTime=1452793217763
2016-01-14 17:40:17,765 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:17,766 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(112)) - Adding #1 tokens and #0 secret keys for NM use for launching container
2016-01-14 17:40:17,766 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(122)) - Putting shuffle token in serviceData
2016-01-14 17:40:17,767 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793217481_0001_00_000001
2016-01-14 17:40:17,767 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:17,768 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793217481_0001_1_02 for vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] numStartedSources: 1 numSources: 1
2016-01-14 17:40:17,768 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] with distanceFromRoot: 3
2016-01-14 17:40:17,771 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: FAE62F9FD3884070A14642DD2A2C76A9 in vertex: C56ED17F7195422F8D2061A379608738
2016-01-14 17:40:17,771 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: C56ED17F7195422F8D2061A379608738 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:17,771 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:17,771 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793217481_0001_1_03, startRequestedTime=1452793217771, startedTime=1452793217771
2016-01-14 17:40:17,771 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:17,773 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793217481_0001_00_000001
2016-01-14 17:40:17,773 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - DAGAppMaster metrics system already initialized!
2016-01-14 17:40:17,774 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793217481_0001_00_000001, launchTime=1452793217774
2016-01-14 17:40:17,774 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:17,774 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:17,777 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000001
2016-01-14 17:40:17,778 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793217481_0001_1_00_000000_0] to container: [container_1452793217481_0001_00_000001]
2016-01-14 17:40:17,778 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:17,778 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793217481_0001_00_000001 given task: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,779 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000001: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,779 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793217481_0001_1_00_000000_0] started. Is using containerId: [container_1452793217481_0001_00_000001] on NM: [127.0.0.1:0]
2016-01-14 17:40:17,779 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:17,780 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=D67FD865B49048F3B8F908CFF8170DDB, taskAttemptId=attempt_1452793217481_0001_1_00_000000_0, startTime=1452793217778, containerId=container_1452793217481_0001_00_000001, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793217481_0001_00_000001/teamcity, completedLogs=
2016-01-14 17:40:17,780 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_00_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:17,780 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_00_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:17 Starting to run new task attempt: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,780 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,781 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:17,781 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:17,781 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:17,782 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:17,782 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:17,782 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:17,782 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: D67FD865B49048F3B8F908CFF8170DDB, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=EC65373C8D9146E3B5B97F9F84B9518E, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=E4BB088620B54C1DA2F37BBDF51DABC7, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:17,783 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:17,783 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,787 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:17,787 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:17,788 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793217481_0001_1_00_000000_0 sent events: (0-1)
2016-01-14 17:40:17,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=EC65373C8D9146E3B5B97F9F84B9518E, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }
2016-01-14 17:40:17,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=E4BB088620B54C1DA2F37BBDF51DABC7, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:17,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:17,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,790 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: INPUT, componentVertexName: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:17,789 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:17,790 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:17,796 INFO  input.MRInput (MRInput.java:initialize(440)) - Using New mapreduce API: false, split information via event: true
2016-01-14 17:40:17,797 INFO  input.MRInput (MRInput.java:initializeInternal(493)) - Initialzed MRInput: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,797 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,800 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:17,801 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:17,801 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:17,803 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:17,814 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: D67FD865B49048F3B8F908CFF8170DDB, ordinal: 0
2016-01-14 17:40:17,814 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: D67FD865B49048F3B8F908CFF8170DDB, mem on start (mb), free: 128, total: 453, max: 672
2016-01-14 17:40:17,814 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:17,815 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:17,815 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:17,816 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:17,816 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:17,816 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.input.MRInput
2016-01-14 17:40:17,816 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:17,816 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:17,817 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:17,817 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 104857600, TotalRequestedScaled: 9.679163076923077E7, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.15
2016-01-14 17:40:17,817 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, EC65373C8D9146E3B5B97F9F84B9518E, org.apache.tez.mapreduce.input.MRInput: requested=0, allocated=0
2016-01-14 17:40:17,819 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, E4BB088620B54C1DA2F37BBDF51DABC7, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=104857600
2016-01-14 17:40:17,819 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:17,819 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: EC65373C8D9146E3B5B97F9F84B9518E being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:17,819 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:17,820 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:17,819 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,820 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:17,820 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,821 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:17,822 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,822 INFO  input.MRInput (MRInput.java:initFromEventInternal(619)) - Initializing RecordReader from event
2016-01-14 17:40:17,823 INFO  input.MRInput (MRInput.java:initFromEventInternal(630)) - Split Details -> SplitClass: org.apache.hadoop.mapred.split.TezGroupedSplit, OldSplit: org.apache.hadoop.mapred.split.TezGroupedSplit@76c2058b
2016-01-14 17:40:17,827 INFO  lib.MRReaderMapred (MRReaderMapred.java:setIncrementalConfigParams(163)) - Processing split: org.apache.hadoop.mapred.split.TezGroupedSplit@76c2058b
2016-01-14 17:40:17,829 INFO  input.MRInput (MRInput.java:initFromEventInternal(634)) - Initialized RecordReader from event
2016-01-14 17:40:17,829 INFO  input.MRInput (MRInput.java:processSplitEvent(586)) - Notifying on RecordReader Initialized
2016-01-14 17:40:17,832 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:17,833 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"] streamed: true, id: EC65373C8D9146E3B5B97F9F84B9518E
2016-01-14 17:40:17,834 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: CoGroup(pearson)[by:[{1}:'l'][numSelfJoins:1]], id: 0F8D87E779ED4BDE997A482210538ABC
2016-01-14 17:40:17,834 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: CoGroup(pearson)[by:[{1}:'l'][numSelfJoins:1]] 0F8D87E779ED4BDE997A482210538ABC
2016-01-14 17:40:17,836 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@7f45c7ec; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@3e48983b; comparator=cascading.tuple.hadoop.util.TupleComparator@1002696f; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:17,836 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:17,837 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:17,837 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:17,871 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 26214396(104857584)
2016-01-14 17:40:17,872 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 100
2016-01-14 17:40:17,872 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 83886080
2016-01-14 17:40:17,872 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 104857600
2016-01-14 17:40:17,872 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 26214396; length = 6553600
2016-01-14 17:40:17,877 INFO  element.TezSourceStage (TezSourceStage.java:prepare(63)) - calling MRInput#start() on: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:17,878 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:17,878 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: D67FD865B49048F3B8F908CFF8170DDB, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:17,885 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,885 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: D67FD865B49048F3B8F908CFF8170DDB, mem on close (mb), free: 160, total: 452, max: 672
2016-01-14 17:40:17,886 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:17,886 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:17,886 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 1676; bufvoid = 104857600
2016-01-14 17:40:17,886 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 26214396(104857584); kvend = 26214260(104857040); length = 137/6553600
2016-01-14 17:40:17,891 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:17 Completed running task attempt: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,894 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793217481_0001_1_00_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:17,896 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793217481_0001_1_00, eventCount=1
2016-01-14 17:40:17,897 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:17,898 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=D67FD865B49048F3B8F908CFF8170DDB, taskAttemptId=attempt_1452793217481_0001_1_00_000000_0, startTime=1452793217778, finishTime=1452793217896, timeTaken=118, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=21, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793217878, Process_Duration=7, Process_End_Time=1452793217885, Read_Duration=3, Tuples_Read=7, Tuples_Written=35, Write_Duration=0, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:17,898 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_00_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:17,898 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17,899 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:17,900 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 28 [[org.apache.tez.common.counters.DAGCounter RACK_LOCAL_TASKS=1][File System Counters FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter SPILLED_RECORDS=35, GC_TIME_MILLIS=21, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0][cascading.flow.SliceCounters Process_Begin_Time=1452793217878, Process_Duration=7, Process_End_Time=1452793217885, Read_Duration=3, Tuples_Read=7, Tuples_Written=35, Write_Duration=0][cascading.flow.StepCounters Tuples_Read=7]]
2016-01-14 17:40:17,900 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_FINISHED]: vertexName=D67FD865B49048F3B8F908CFF8170DDB, taskId=task_1452793217481_0001_1_00_000000, startTime=1452793217778, finishTime=1452793217899, timeTaken=121, status=SUCCEEDED, successfulAttemptID=attempt_1452793217481_0001_1_00_000000_0, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=21, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793217878, Process_Duration=7, Process_End_Time=1452793217885, Read_Duration=3, Tuples_Read=7, Tuples_Written=35, Write_Duration=0, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:17,900 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:17,900 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:17,901 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:17,901 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_00_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:17,902 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000001
2016-01-14 17:40:17,902 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] : 1
2016-01-14 17:40:17,902 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:17,904 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:17,905 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_FINISHED]: vertexName=D67FD865B49048F3B8F908CFF8170DDB, vertexId=vertex_1452793217481_0001_1_00, initRequestedTime=1452793217695, initedTime=1452793217749, startRequestedTime=1452793217753, startedTime=1452793217753, finishTime=1452793217902, timeTaken=149, status=SUCCEEDED, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1121, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=21, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793217878, Process_Duration=7, Process_End_Time=1452793217885, Read_Duration=3, Tuples_Read=7, Tuples_Written=35, Write_Duration=0, cascading.flow.StepCounters, Tuples_Read=7, vertexStats=firstTaskStartTime=1452793217778, firstTasksToStart=[ task_1452793217481_0001_1_00_000000 ], lastTaskFinishTime=1452793217896, lastTasksToFinish=[ task_1452793217481_0001_1_00_000000 ], minTaskDuration=118, maxTaskDuration=118, avgTaskDuration=118.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793217481_0001_1_00_000000 ], longestDurationTasks=[ task_1452793217481_0001_1_00_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:17,905 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:17,906 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:17,906 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:17,906 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793217481_0001_00_000001, stoppedTime=1452793217906, exitStatus=0
2016-01-14 17:40:17,907 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:17,907 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] attempt: attempt_1452793217481_0001_1_00_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:17,907 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:17,908 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:17,908 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB] completed., numCompletedVertices=1, numSuccessfulVertices=1, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:17,909 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793217481_0001_00_000001. Relying on regular task shutdown for it to end
2016-01-14 17:40:17,909 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_STARTED]: vertexName=E4BB088620B54C1DA2F37BBDF51DABC7, taskId=task_1452793217481_0001_1_01_000000, scheduledTime=1452793217909, launchTime=1452793217909
2016-01-14 17:40:17,909 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:17,910 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_01_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:17,910 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:17,910 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:17,911 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: E4BB088620B54C1DA2F37BBDF51DABC7, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=D67FD865B49048F3B8F908CFF8170DDB, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=FAE62F9FD3884070A14642DD2A2C76A9, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:17,911 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_01_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:17,911 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:17,912 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793217481_0001_00_000001 is valid, but no longer registered, and will be killed
2016-01-14 17:40:17,912 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000001: 10 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:17,913 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793217481_0001_00_000001, Exiting
2016-01-14 17:40:17,913 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793217481_0001_00_000001
2016-01-14 17:40:17,913 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793217481_0001_00_000001
2016-01-14 17:40:17,914 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793217481_0001_00_000002
2016-01-14 17:40:17,915 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:17,916 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793217481_0001_00_000001 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:17,916 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793217481_0001_00_000001 completed successfully
2016-01-14 17:40:17,917 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793217481_0001_00_000002
2016-01-14 17:40:17,918 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000001 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:17,918 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:17,920 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:17,921 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:17,921 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793217481_0001_00_000002, launchTime=1452793217920
2016-01-14 17:40:17,922 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000002
2016-01-14 17:40:17,922 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793217481_0001_1_01_000000_0] to container: [container_1452793217481_0001_00_000002]
2016-01-14 17:40:17,922 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:17,923 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793217481_0001_1_01_000000_0] started. Is using containerId: [container_1452793217481_0001_00_000002] on NM: [127.0.0.1:0]
2016-01-14 17:40:17,923 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:17,923 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=E4BB088620B54C1DA2F37BBDF51DABC7, taskAttemptId=attempt_1452793217481_0001_1_01_000000_0, startTime=1452793217923, containerId=container_1452793217481_0001_00_000002, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793217481_0001_00_000002/teamcity, completedLogs=
2016-01-14 17:40:17,923 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_01_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:17,923 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_01_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:17,923 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793217481_0001_00_000002 given task: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:17,924 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000002: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:17 Starting to run new task attempt: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:17,924 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:17,925 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:17,925 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:17,925 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:17,925 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:17,925 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:17,926 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:17,926 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: E4BB088620B54C1DA2F37BBDF51DABC7, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=D67FD865B49048F3B8F908CFF8170DDB, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=FAE62F9FD3884070A14642DD2A2C76A9, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:17,926 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:17,927 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:17,928 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:17,928 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:17,927 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793217481_0001_1_01_000000_0 sent events: (0-1)
2016-01-14 17:40:17,929 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:17,929 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=FAE62F9FD3884070A14642DD2A2C76A9, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:17,929 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=D67FD865B49048F3B8F908CFF8170DDB, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:17,930 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:17,930 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:17,930 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:17,931 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,933 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:17,933 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:17,933 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:17,934 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:17,934 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:17,934 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,934 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:17,935 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,944 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: E4BB088620B54C1DA2F37BBDF51DABC7, ordinal: 1
2016-01-14 17:40:17,944 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: E4BB088620B54C1DA2F37BBDF51DABC7, mem on start (mb), free: 152, total: 452, max: 672
2016-01-14 17:40:17,945 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:17,945 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:17,945 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:17,945 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:17,946 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:17,946 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:17,946 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:17,947 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:17,947 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:17,947 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, FAE62F9FD3884070A14642DD2A2C76A9, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:17,948 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, D67FD865B49048F3B8F908CFF8170DDB, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:17,948 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:17,948 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: D67FD865B49048F3B8F908CFF8170DDB being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:17,948 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,949 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:17,950 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:17,952 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:17,953 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:17,953 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: D67FD865B49048F3B8F908CFF8170DDB with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:17,953 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:17,953 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:17,954 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:17,956 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,957 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:17,957 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:17,958 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793217481_0001_1_00_000000_0_10015, runDuration: 96998, ]
2016-01-14 17:40:17,959 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793217481_0001_1_00_000000_0_10015] to DISK_DIRECT, CompressedSize=1089, DecompressedSize=1085,EndTime=1452793217959, TimeTaken=1, Rate=1.04 MB/s
2016-01-14 17:40:17,960 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,960 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:17,960 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [D67FD865B49048F3B8F908CFF8170DDB] #4 in 2ms
2016-01-14 17:40:17,960 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [D67FD865B49048F3B8F908CFF8170DDB] #4, status:false, isInterrupted:false
2016-01-14 17:40:17,961 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:17,962 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 1089 bytes from disk
2016-01-14 17:40:17,962 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:17,962 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:17,962 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: CoGroup(pearson)[by:[{1}:'l'][numSelfJoins:1]]
2016-01-14 17:40:17,963 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 1061 bytes
2016-01-14 17:40:17,963 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:17,963 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:17,964 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: CoGroup(pearson)[by:[{1}:'l'][numSelfJoins:1]] streamed: true, id: 0F8D87E779ED4BDE997A482210538ABC
2016-01-14 17:40:17,964 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(pearson)[by:[{?}:ALL]], id: 2B35EB80AB754016B6590AFB37ABABB0
2016-01-14 17:40:17,964 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(pearson)[by:[{?}:ALL]] 2B35EB80AB754016B6590AFB37ABABB0
2016-01-14 17:40:17,966 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@5959b1ba; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@fa3582a; comparator=cascading.tuple.hadoop.util.TupleComparator@2cf1356b; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:17,966 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:17,966 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:17,966 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:18,010 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:18,010 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:18,010 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:18,010 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:18,010 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:18,011 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: CoGroup(pearson)[by:[{1}:'l'][numSelfJoins:1]] 0F8D87E779ED4BDE997A482210538ABC, for 1 inputs
2016-01-14 17:40:18,012 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(89)) - attempting to load codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:18,012 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(94)) - found codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:18,013 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: E4BB088620B54C1DA2F37BBDF51DABC7, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:18,014 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@30754c09; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@64d2fbb9
2016-01-14 17:40:18,044 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:18,044 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: E4BB088620B54C1DA2F37BBDF51DABC7, mem on close (mb), free: 85, total: 452, max: 672
2016-01-14 17:40:18,045 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: D67FD865B49048F3B8F908CFF8170DDB
2016-01-14 17:40:18,048 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:18,049 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:18,049 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:18,049 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:18,049 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:18,050 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:18,050 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:18,050 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 13104; bufvoid = 66060288
2016-01-14 17:40:18,050 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514376(66057504); length = 693/4128768
2016-01-14 17:40:18,067 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:18 Completed running task attempt: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:18,071 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793217481_0001_1_01_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:18,072 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793217481_0001_1_01, eventCount=1
2016-01-14 17:40:18,073 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793218013, Process_Duration=31, Process_End_Time=1452793218044, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=10]]
2016-01-14 17:40:18,073 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:18,073 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:18,074 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:18,074 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=E4BB088620B54C1DA2F37BBDF51DABC7, taskAttemptId=attempt_1452793217481_0001_1_01_000000_0, startTime=1452793217923, finishTime=1452793218073, timeTaken=150, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218013, Process_Duration=31, Process_End_Time=1452793218044, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=10
2016-01-14 17:40:18,074 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000002
2016-01-14 17:40:18,075 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_01_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:18,075 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:18,075 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:18,076 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:18,077 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_FINISHED]: vertexName=E4BB088620B54C1DA2F37BBDF51DABC7, taskId=task_1452793217481_0001_1_01_000000, startTime=1452793217923, finishTime=1452793218076, timeTaken=153, status=SUCCEEDED, successfulAttemptID=attempt_1452793217481_0001_1_01_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218013, Process_Duration=31, Process_End_Time=1452793218044, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=10
2016-01-14 17:40:18,077 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_01_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:18,077 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] : 1
2016-01-14 17:40:18,077 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:18,078 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:18,080 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_FINISHED]: vertexName=E4BB088620B54C1DA2F37BBDF51DABC7, vertexId=vertex_1452793217481_0001_1_01, initRequestedTime=1452793217697, initedTime=1452793217714, startRequestedTime=1452793217759, startedTime=1452793217759, finishTime=1452793218077, timeTaken=318, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218013, Process_Duration=31, Process_End_Time=1452793218044, Read_Duration=3, Tuples_Read=35, Tuples_Written=174, Write_Duration=10, vertexStats=firstTaskStartTime=1452793217923, firstTasksToStart=[ task_1452793217481_0001_1_01_000000 ], lastTaskFinishTime=1452793218073, lastTasksToFinish=[ task_1452793217481_0001_1_01_000000 ], minTaskDuration=150, maxTaskDuration=150, avgTaskDuration=150.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793217481_0001_1_01_000000 ], longestDurationTasks=[ task_1452793217481_0001_1_01_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:18,080 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:18,081 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:18,081 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:18,081 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793217481_0001_00_000002, stoppedTime=1452793218081, exitStatus=0
2016-01-14 17:40:18,082 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:18,082 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] attempt: attempt_1452793217481_0001_1_01_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:18,082 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,083 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:18,083 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7] completed., numCompletedVertices=2, numSuccessfulVertices=2, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:18,084 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793217481_0001_00_000002. Relying on regular task shutdown for it to end
2016-01-14 17:40:18,084 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:18,084 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_STARTED]: vertexName=FAE62F9FD3884070A14642DD2A2C76A9, taskId=task_1452793217481_0001_1_02_000000, scheduledTime=1452793218084, launchTime=1452793218084
2016-01-14 17:40:18,084 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_02_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:18,084 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:18,085 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:18,085 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: FAE62F9FD3884070A14642DD2A2C76A9, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=E4BB088620B54C1DA2F37BBDF51DABC7, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=C56ED17F7195422F8D2061A379608738, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:18,086 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_02_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:18,085 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793217481_0001_00_000002 is valid, but no longer registered, and will be killed
2016-01-14 17:40:18,086 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:18,087 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000002: 12 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:18,087 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793217481_0001_00_000002, Exiting
2016-01-14 17:40:18,088 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793217481_0001_00_000002
2016-01-14 17:40:18,088 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793217481_0001_00_000002
2016-01-14 17:40:18,089 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793217481_0001_00_000002 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:18,089 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793217481_0001_00_000002 completed successfully
2016-01-14 17:40:18,091 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793217481_0001_00_000003
2016-01-14 17:40:18,091 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:18,091 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000002 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:18,093 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793217481_0001_00_000003
2016-01-14 17:40:18,094 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:18,097 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:18,097 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:18,097 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793217481_0001_00_000003, launchTime=1452793218097
2016-01-14 17:40:18,098 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000003
2016-01-14 17:40:18,100 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793217481_0001_1_02_000000_0] to container: [container_1452793217481_0001_00_000003]
2016-01-14 17:40:18,100 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:18,100 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793217481_0001_00_000003 given task: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,101 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000003: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,101 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793217481_0001_1_02_000000_0] started. Is using containerId: [container_1452793217481_0001_00_000003] on NM: [127.0.0.1:0]
2016-01-14 17:40:18,101 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:18,101 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=FAE62F9FD3884070A14642DD2A2C76A9, taskAttemptId=attempt_1452793217481_0001_1_02_000000_0, startTime=1452793218101, containerId=container_1452793217481_0001_00_000003, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793217481_0001_00_000003/teamcity, completedLogs=
2016-01-14 17:40:18,102 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_02_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:18,102 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_02_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:18 Starting to run new task attempt: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,103 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,103 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:18,103 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:18,103 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:18,104 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:18,104 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:18,104 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:18,105 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: FAE62F9FD3884070A14642DD2A2C76A9, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=E4BB088620B54C1DA2F37BBDF51DABC7, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=C56ED17F7195422F8D2061A379608738, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:18,105 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:18,106 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,106 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:18,106 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:18,107 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:18,108 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=E4BB088620B54C1DA2F37BBDF51DABC7, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:18,108 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793217481_0001_1_02_000000_0 sent events: (0-1)
2016-01-14 17:40:18,109 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:18,110 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,111 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=C56ED17F7195422F8D2061A379608738, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:18,112 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:18,112 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: C56ED17F7195422F8D2061A379608738
2016-01-14 17:40:18,113 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:18,114 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:18,115 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:18,115 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,116 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,117 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:18,117 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: C56ED17F7195422F8D2061A379608738
2016-01-14 17:40:18,118 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: C56ED17F7195422F8D2061A379608738
2016-01-14 17:40:18,123 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: FAE62F9FD3884070A14642DD2A2C76A9, ordinal: 2
2016-01-14 17:40:18,123 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: FAE62F9FD3884070A14642DD2A2C76A9, mem on start (mb), free: 79, total: 452, max: 672
2016-01-14 17:40:18,123 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:18,123 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:18,123 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:18,124 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:18,124 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:18,124 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:18,124 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:18,125 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:18,125 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:18,125 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, E4BB088620B54C1DA2F37BBDF51DABC7, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:18,125 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, C56ED17F7195422F8D2061A379608738, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:18,126 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:18,126 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: E4BB088620B54C1DA2F37BBDF51DABC7 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:18,126 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:18,127 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:18,126 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,128 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:18,129 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:18,130 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: E4BB088620B54C1DA2F37BBDF51DABC7 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:18,130 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:18,130 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:18,131 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:18,131 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,132 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:18,132 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,136 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793217481_0001_1_01_000000_0_10017, runDuration: 138593, ]
2016-01-14 17:40:18,139 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793217481_0001_1_01_000000_0_10017] to DISK_DIRECT, CompressedSize=7519, DecompressedSize=7515,EndTime=1452793218139, TimeTaken=2, Rate=3.59 MB/s
2016-01-14 17:40:18,139 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,140 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.01 MB/s)
2016-01-14 17:40:18,140 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(pearson)[by:[{?}:ALL]]
2016-01-14 17:40:18,140 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [E4BB088620B54C1DA2F37BBDF51DABC7] #5 in 3ms
2016-01-14 17:40:18,141 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [E4BB088620B54C1DA2F37BBDF51DABC7] #5, status:false, isInterrupted:false
2016-01-14 17:40:18,142 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:18,142 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(pearson)[by:[{?}:ALL]] streamed: true, id: 2B35EB80AB754016B6590AFB37ABABB0
2016-01-14 17:40:18,142 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 7519 bytes from disk
2016-01-14 17:40:18,143 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:18,143 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(pearson)[by:[{2}:'n1', 'n2']], id: 484EBB254C414B04988922244601A117
2016-01-14 17:40:18,144 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:18,145 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(pearson)[by:[{2}:'n1', 'n2']] 484EBB254C414B04988922244601A117
2016-01-14 17:40:18,145 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 7443 bytes
2016-01-14 17:40:18,146 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,147 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:18,147 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@eab96ab; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@2330bc13; comparator=cascading.tuple.hadoop.util.TupleComparator@548b9571; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:18,148 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:18,148 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:18,148 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:18,170 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:18,170 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:18,170 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:18,170 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:18,171 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:18,171 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(pearson)[by:[{?}:ALL]] 2B35EB80AB754016B6590AFB37ABABB0, for 1 inputs
2016-01-14 17:40:18,172 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: FAE62F9FD3884070A14642DD2A2C76A9, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:18,172 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@28394fd9; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@4aa94430
2016-01-14 17:40:18,191 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,191 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: FAE62F9FD3884070A14642DD2A2C76A9, mem on close (mb), free: 200, total: 452, max: 672
2016-01-14 17:40:18,192 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: E4BB088620B54C1DA2F37BBDF51DABC7
2016-01-14 17:40:18,192 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:18,193 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:18,193 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:18,193 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:18,193 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:18,194 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:18,194 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:18,194 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 6291; bufvoid = 66060288
2016-01-14 17:40:18,194 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514724(66058896); length = 345/4128768
2016-01-14 17:40:18,198 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:18 Completed running task attempt: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,201 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793217481_0001_1_02_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:18,202 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793217481_0001_1_02, eventCount=1
2016-01-14 17:40:18,202 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=14, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=11, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793218172, Process_Duration=19, Process_End_Time=1452793218191, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=3]]
2016-01-14 17:40:18,202 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:18,203 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=FAE62F9FD3884070A14642DD2A2C76A9, taskAttemptId=attempt_1452793217481_0001_1_02_000000_0, startTime=1452793218101, finishTime=1452793218202, timeTaken=101, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=14, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=11, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218172, Process_Duration=19, Process_End_Time=1452793218191, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=3
2016-01-14 17:40:18,204 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_02_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:18,204 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:18,204 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:18,205 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18,205 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:18,205 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000003
2016-01-14 17:40:18,205 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:18,206 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_FINISHED]: vertexName=FAE62F9FD3884070A14642DD2A2C76A9, taskId=task_1452793217481_0001_1_02_000000, startTime=1452793218101, finishTime=1452793218205, timeTaken=104, status=SUCCEEDED, successfulAttemptID=attempt_1452793217481_0001_1_02_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=14, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=11, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218172, Process_Duration=19, Process_End_Time=1452793218191, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=3
2016-01-14 17:40:18,206 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_02_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:18,207 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] : 1
2016-01-14 17:40:18,207 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:18,208 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:18,208 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_FINISHED]: vertexName=FAE62F9FD3884070A14642DD2A2C76A9, vertexId=vertex_1452793217481_0001_1_02, initRequestedTime=1452793217715, initedTime=1452793217725, startRequestedTime=1452793217763, startedTime=1452793217763, finishTime=1452793218207, timeTaken=444, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=4376, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=14, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=6291, OUTPUT_BYTES_WITH_OVERHEAD=4340, OUTPUT_BYTES_PHYSICAL=4344, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=11, MERGE_PHASE_TIME=17, FIRST_EVENT_RECEIVED=8, LAST_EVENT_RECEIVED=8, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218172, Process_Duration=19, Process_End_Time=1452793218191, Read_Duration=9, Tuples_Read=174, Tuples_Written=87, Write_Duration=3, vertexStats=firstTaskStartTime=1452793218101, firstTasksToStart=[ task_1452793217481_0001_1_02_000000 ], lastTaskFinishTime=1452793218202, lastTasksToFinish=[ task_1452793217481_0001_1_02_000000 ], minTaskDuration=101, maxTaskDuration=101, avgTaskDuration=101.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793217481_0001_1_02_000000 ], longestDurationTasks=[ task_1452793217481_0001_1_02_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:18,209 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:18,209 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:18,209 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793217481_0001_00_000003, stoppedTime=1452793218209, exitStatus=0
2016-01-14 17:40:18,209 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:18,209 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] attempt: attempt_1452793217481_0001_1_02_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:18,209 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: C56ED17F7195422F8D2061A379608738
2016-01-14 17:40:18,210 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:18,210 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9] completed., numCompletedVertices=3, numSuccessfulVertices=3, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:18,210 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793217481_0001_00_000003. Relying on regular task shutdown for it to end
2016-01-14 17:40:18,210 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_STARTED]: vertexName=C56ED17F7195422F8D2061A379608738, taskId=task_1452793217481_0001_1_03_000000, scheduledTime=1452793218210, launchTime=1452793218210
2016-01-14 17:40:18,210 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_03_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:18,210 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:18,210 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:18,211 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: C56ED17F7195422F8D2061A379608738, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=FAE62F9FD3884070A14642DD2A2C76A9, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=1D79E0D7E49D4EA999A42FBB82B8CDDD, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:18,209 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:18,211 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_03_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:18,211 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:18,211 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:18,212 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793217481_0001_00_000004
2016-01-14 17:40:18,212 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000004 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:18,214 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793217481_0001_00_000004
2016-01-14 17:40:18,214 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - TezTask metrics system already initialized!
2016-01-14 17:40:18,215 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793217481_0001_00_000004, launchTime=1452793218215
2016-01-14 17:40:18,215 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:18,215 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000004 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:18,216 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793217481_0001_00_000003 is valid, but no longer registered, and will be killed
2016-01-14 17:40:18,216 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000003: 10 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:18,217 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793217481_0001_00_000003, Exiting
2016-01-14 17:40:18,217 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793217481_0001_00_000003
2016-01-14 17:40:18,217 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793217481_0001_00_000003
2016-01-14 17:40:18,218 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793217481_0001_00_000003 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:18,218 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793217481_0001_00_000003 completed successfully
2016-01-14 17:40:18,218 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000003 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:18,218 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000004
2016-01-14 17:40:18,220 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793217481_0001_1_03_000000_0] to container: [container_1452793217481_0001_00_000004]
2016-01-14 17:40:18,220 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000004 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:18,221 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793217481_0001_00_000004 given task: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,221 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000004: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18 Starting to run new task attempt: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,222 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,222 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:18,222 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:18,222 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:18,222 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:18,222 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:18,223 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:18,221 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793217481_0001_1_03_000000_0] started. Is using containerId: [container_1452793217481_0001_00_000004] on NM: [127.0.0.1:0]
2016-01-14 17:40:18,223 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A] pearson/(1/1) ...mposite/pearson/composite, VertexName: C56ED17F7195422F8D2061A379608738, VertexParallelism: 1, TaskAttemptID:attempt_1452793217481_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=FAE62F9FD3884070A14642DD2A2C76A9, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=1D79E0D7E49D4EA999A42FBB82B8CDDD, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:18,223 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:18,223 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:18,223 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=C56ED17F7195422F8D2061A379608738, taskAttemptId=attempt_1452793217481_0001_1_03_000000_0, startTime=1452793218221, containerId=container_1452793217481_0001_00_000004, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793217481_0001_00_000004/teamcity, completedLogs=
2016-01-14 17:40:18,224 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_03_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:18,224 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_03_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:18,224 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,229 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:18,229 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:18,229 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793217481_0001_1_03_000000_0 sent events: (0-1)
2016-01-14 17:40:18,231 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:18,233 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=1D79E0D7E49D4EA999A42FBB82B8CDDD, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }
2016-01-14 17:40:18,233 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=FAE62F9FD3884070A14642DD2A2C76A9, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:18,234 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:18,234 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:18,235 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,235 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:18,234 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: 1D79E0D7E49D4EA999A42FBB82B8CDDD
2016-01-14 17:40:18,238 INFO  output.MROutput (MROutput.java:initialize(347)) - Initializing Simple Output
2016-01-14 17:40:18,238 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: OUTPUT, componentVertexName: 1D79E0D7E49D4EA999A42FBB82B8CDDD
2016-01-14 17:40:18,236 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:18,238 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:18,239 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,239 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,248 INFO  output.MROutput (MROutput.java:initialize(418)) - Initialized Simple Output, using_new_api: false
2016-01-14 17:40:18,249 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: 1D79E0D7E49D4EA999A42FBB82B8CDDD
2016-01-14 17:40:18,252 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: C56ED17F7195422F8D2061A379608738, ordinal: 3
2016-01-14 17:40:18,252 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: C56ED17F7195422F8D2061A379608738, mem on start (mb), free: 192, total: 452, max: 672
2016-01-14 17:40:18,252 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:18,252 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:18,252 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:18,252 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:18,253 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:18,253 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.output.MROutput
2016-01-14 17:40:18,253 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:18,253 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:18,253 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:18,253 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 634178752, TotalRequestedScaled: 5.853957710769231E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.90
2016-01-14 17:40:18,253 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, 1D79E0D7E49D4EA999A42FBB82B8CDDD, org.apache.tez.mapreduce.output.MROutput: requested=0, allocated=0
2016-01-14 17:40:18,254 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, FAE62F9FD3884070A14642DD2A2C76A9, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=472110858
2016-01-14 17:40:18,254 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:18,254 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: FAE62F9FD3884070A14642DD2A2C76A9 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:18,254 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:18,254 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:18,254 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,255 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:18,255 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:18,256 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: FAE62F9FD3884070A14642DD2A2C76A9 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:18,256 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=472110858. Updated to: ShuffleMem=472110858, postMergeMem: 0
2016-01-14 17:40:18,256 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=472110858, maxSingleShuffleLimit=118027712, mergeThreshold=424899744, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:18,256 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:18,258 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,258 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:18,260 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,261 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793217481_0001_1_02_000000_0_10021, runDuration: 87338, ]
2016-01-14 17:40:18,263 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793217481_0001_1_02_000000_0_10021] to DISK_DIRECT, CompressedSize=4344, DecompressedSize=4340,EndTime=1452793218263, TimeTaken=1, Rate=4.14 MB/s
2016-01-14 17:40:18,263 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,264 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(pearson)[by:[{2}:'n1', 'n2']]
2016-01-14 17:40:18,264 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:18,264 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [FAE62F9FD3884070A14642DD2A2C76A9] #6 in 2ms
2016-01-14 17:40:18,265 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [FAE62F9FD3884070A14642DD2A2C76A9] #6, status:false, isInterrupted:false
2016-01-14 17:40:18,265 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(pearson)[by:[{2}:'n1', 'n2']] streamed: true, id: 484EBB254C414B04988922244601A117
2016-01-14 17:40:18,266 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:18,267 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testPearsonDistanceComposite/pearson/composite"], id: 1D79E0D7E49D4EA999A42FBB82B8CDDD
2016-01-14 17:40:18,267 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 4344 bytes from disk
2016-01-14 17:40:18,267 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:18,267 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:18,267 INFO  element.TezSinkStage (TezSinkStage.java:prepare(57)) - calling MROutput#start() on: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testPearsonDistanceComposite/pearson/composite"]
2016-01-14 17:40:18,268 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(pearson)[by:[{2}:'n1', 'n2']] 484EBB254C414B04988922244601A117, for 1 inputs
2016-01-14 17:40:18,268 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 4301 bytes
2016-01-14 17:40:18,269 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,269 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: C56ED17F7195422F8D2061A379608738, all 1 inputs ready in: 00:00:00.001
2016-01-14 17:40:18,269 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:18,270 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@72f69182; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@19cb28c5
2016-01-14 17:40:18,290 INFO  output.MROutput (MROutput.java:flush(534)) - Flushing Simple Output
2016-01-14 17:40:18,290 INFO  output.MROutput (MROutput.java:flush(544)) - Flushed Simple Output
2016-01-14 17:40:18,291 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:canCommit(353)) - Commit go/no-go request from attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,291 INFO  impl.TaskImpl (TaskImpl.java:canCommit(750)) - attempt_1452793217481_0001_1_03_000000_0 given a go for committing the task output.
2016-01-14 17:40:18,292 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,292 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: C56ED17F7195422F8D2061A379608738, mem on close (mb), free: 189, total: 452, max: 672
2016-01-14 17:40:18,292 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: FAE62F9FD3884070A14642DD2A2C76A9
2016-01-14 17:40:18,292 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:18,293 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:18,293 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:18,293 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:18,293 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:18 Completed running task attempt: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,294 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793217481_0001_1_03_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:18,295 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=C56ED17F7195422F8D2061A379608738, taskAttemptId=attempt_1452793217481_0001_1_03_000000_0, startTime=1452793218221, finishTime=1452793218295, timeTaken=74, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=1369, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=8, MERGE_PHASE_TIME=13, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218269, Process_Duration=23, Process_End_Time=1452793218292, Read_Duration=5, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:18,296 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793217481_0001_1_03_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:18,296 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:18,296 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18,296 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:18,297 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:TASK_FINISHED]: vertexName=C56ED17F7195422F8D2061A379608738, taskId=task_1452793217481_0001_1_03_000000, startTime=1452793218221, finishTime=1452793218296, timeTaken=75, status=SUCCEEDED, successfulAttemptID=attempt_1452793217481_0001_1_03_000000_0, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=1369, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=8, MERGE_PHASE_TIME=13, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218269, Process_Duration=23, Process_End_Time=1452793218292, Read_Duration=5, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:18,297 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:18,297 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793217481_0001_1_03_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:18,297 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] : 1
2016-01-14 17:40:18,298 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:18,300 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:VERTEX_FINISHED]: vertexName=C56ED17F7195422F8D2061A379608738, vertexId=vertex_1452793217481_0001_1_03, initRequestedTime=1452793217726, initedTime=1452793217744, startRequestedTime=1452793217771, startedTime=1452793217771, finishTime=1452793218298, timeTaken=527, status=SUCCEEDED, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=1369, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=8, MERGE_PHASE_TIME=13, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793218269, Process_Duration=23, Process_End_Time=1452793218292, Read_Duration=5, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21, vertexStats=firstTaskStartTime=1452793218221, firstTasksToStart=[ task_1452793217481_0001_1_03_000000 ], lastTaskFinishTime=1452793218295, lastTasksToFinish=[ task_1452793217481_0001_1_03_000000 ], minTaskDuration=74, maxTaskDuration=74, avgTaskDuration=74.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793217481_0001_1_03_000000 ], longestDurationTasks=[ task_1452793217481_0001_1_03_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:18,300 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:18,300 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000004 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:18,300 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 46 [[File System Counters FILE_BYTES_READ=4376, FILE_BYTES_WRITTEN=1369, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=4344, SHUFFLE_BYTES=4344, SHUFFLE_BYTES_DECOMPRESSED=4340, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=4344, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=8, MERGE_PHASE_TIME=13, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793218269, Process_Duration=23, Process_End_Time=1452793218292, Read_Duration=5, Tuples_Read=87, Tuples_Written=21, Write_Duration=0][cascading.flow.StepCounters Tuples_Written=21]]
2016-01-14 17:40:18,301 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793217481_0001_00_000004, stoppedTime=1452793218301, exitStatus=0
2016-01-14 17:40:18,301 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:18,301 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000004 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:18,301 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:18,301 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738] completed., numCompletedVertices=4, numSuccessfulVertices=4, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:18,302 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:18,302 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(831)) - Calling DAG commit/abort for dag: dag_1452793217481_0001_1
2016-01-14 17:40:18,302 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:DAG_COMMIT_STARTED]: dagID=dag_1452793217481_0001_1
2016-01-14 17:40:18,302 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793217481_0001_1_00 [D67FD865B49048F3B8F908CFF8170DDB]
2016-01-14 17:40:18,302 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:18,303 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793217481_0001_00_000004
2016-01-14 17:40:18,303 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793217481_0001_00_000004 is valid, but no longer registered, and will be killed
2016-01-14 17:40:18,303 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793217481_0001_00_000004: 0 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:18,302 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793217481_0001_1_01 [E4BB088620B54C1DA2F37BBDF51DABC7]
2016-01-14 17:40:18,303 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793217481_0001_1_02 [FAE62F9FD3884070A14642DD2A2C76A9]
2016-01-14 17:40:18,303 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(892)) - Committing output: 1D79E0D7E49D4EA999A42FBB82B8CDDD for vertex: vertex_1452793217481_0001_1_03 [C56ED17F7195422F8D2061A379608738]
2016-01-14 17:40:18,305 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793217481_0001_00_000004, Exiting
2016-01-14 17:40:18,305 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793217481_0001_00_000004
2016-01-14 17:40:18,305 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793217481_0001_00_000004
2016-01-14 17:40:18,306 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793217481_0001_00_000004 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:18,306 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793217481_0001_00_000004 completed successfully
2016-01-14 17:40:18,318 INFO  recovery.RecoveryService (RecoveryService.java:handle(306)) - DAG completed, dagId=dag_1452793217481_0001_1, queueSize=0
2016-01-14 17:40:18,318 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_FINISHED to history file
2016-01-14 17:40:18,318 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793217481_0001_1][Event:DAG_FINISHED]: dagId=dag_1452793217481_0001_1, startTime=1452793217694, finishTime=1452793218312, timeTaken=618, status=SUCCEEDED, diagnostics=, counters=Counters: 55, org.apache.tez.common.counters.DAGCounter, NUM_SUCCEEDED_TASKS=4, TOTAL_LAUNCHED_TASKS=4, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=14386, FILE_BYTES_WRITTEN=14417, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=111, REDUCE_INPUT_RECORDS=296, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=592, NUM_SHUFFLED_INPUTS=3, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=3, GC_TIME_MILLIS=35, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=1897922560, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=317, OUTPUT_BYTES=21071, OUTPUT_BYTES_WITH_OVERHEAD=12940, OUTPUT_BYTES_PHYSICAL=12952, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=12952, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=12952, SHUFFLE_BYTES_DECOMPRESSED=12940, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=12952, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=26, MERGE_PHASE_TIME=40, FIRST_EVENT_RECEIVED=22, LAST_EVENT_RECEIVED=22, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=5811172872332, Process_Duration=80, Process_End_Time=5811172872412, Read_Duration=20, Tuples_Read=303, Tuples_Written=317, Write_Duration=13, cascading.flow.StepCounters, Tuples_Read=7, Tuples_Written=21
2016-01-14 17:40:18,318 INFO  impl.DAGImpl (DAGImpl.java:finished(1180)) - DAG: dag_1452793217481_0001_1 finished with state: SUCCEEDED
2016-01-14 17:40:18,319 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793217481_0001_1 transitioned from RUNNING to SUCCEEDED due to event DAG_VERTEX_COMPLETED
2016-01-14 17:40:18,319 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(242)) - Ignoring stop request for containerId: container_1452793217481_0001_00_000004
2016-01-14 17:40:18,319 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793217481_0001_00_000004 transitioned from STOP_REQUESTED to COMPLETED via event C_COMPLETED
2016-01-14 17:40:18 Completed Dag: dag_1452793217481_0001_1
2016-01-14 17:40:18,321 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(579)) - DAG completed, dagId=dag_1452793217481_0001_1, dagState=SUCCEEDED
2016-01-14 17:40:18,322 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793217481_0001_1_post
2016-01-14 17:40:18,323 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:18,324 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(625)) - Waiting for next DAG to be submitted.
2016-01-14 17:40:18,327 INFO  client.TezClient (TezClient.java:stop(495)) - Shutting down Tez Session, sessionName=(1/1) ...mposite/pearson/composite, applicationId=application_1452793217481_0001
2016-01-14 17:40:18,330 INFO  client.DAGClientHandler (DAGClientHandler.java:shutdownAM(117)) - Received message to shutdown AM
2016-01-14 17:40:18,330 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:setShouldUnregisterFlag(580)) - TaskScheduler notified that it should unregister from RM
2016-01-14 17:40:18,330 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdownTezAM(1088)) - No current running DAG, shutting down the AM
2016-01-14 17:40:18,330 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(692)) - DAGAppMasterShutdownHandler invoked
2016-01-14 17:40:18,330 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(701)) - Handling DAGAppMaster shutdown
2016-01-14 17:40:18,331 INFO  app.DAGAppMaster (DAGAppMaster.java:run(721)) - Sleeping for 5 seconds before shutting down
2016-01-14 17:40:18,331 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStop(79)) - Stopping HistoryEventHandler
2016-01-14 17:40:18,332 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(189)) - Stopping RecoveryService
2016-01-14 17:40:18,332 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(192)) - Handle the remaining events in queue, queue size=0
2016-01-14 17:40:18,332 INFO  recovery.RecoveryService (RecoveryService.java:run(166)) - EventQueue take interrupted. Returning
2016-01-14 17:40:18,333 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(214)) - Closing Summary Stream
2016-01-14 17:40:18,333 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStop(118)) - Stopping SimpleHistoryLoggingService, eventQueueBacklog=0
2016-01-14 17:40:18,333 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:run(105)) - EventQueue take interrupted. Returning
2016-01-14 17:40:18,340 INFO  app.DAGAppMaster (DAGAppMaster.java:run(1773)) - Completed deletion of tez scratch data dir, path=file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5EB42626A054E8F9A2E57644234EF5D/B755B348C998446699B55356B1BDDB1A/.tez/application_1452793217481_0001
2016-01-14 17:40:18,340 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [pearson]  completed in: 00:00.883
2016-01-14 17:40:18,427 INFO  cascading.PlatformTestCase (PlatformTestCase.java:tearDown(189)) - copying to local /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output//hadoop2-tez/distanceusecaseplatform/testPearsonDistanceComposite/pearson/composite
2016-01-14 17:40:18,471 INFO  util.Util (Util.java:findMainClass(1327)) - using default application jar, may cause class not found exceptions on the cluster
2016-01-14 17:40:18,471 INFO  planner.Hadoop2TezPlanner (Hadoop2TezPlanner.java:initialize(149)) - using application jar: /opt/concurrent/home/teamcity/.gradle/caches/modules-2/files-2.1/cascading/cascading-hadoop2-tez/3.0.4/561447ee876073bb61fddc52e2ad157f45c8edd/cascading-hadoop2-tez-3.0.4.jar
2016-01-14 17:40:18,557 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] executed rule registry: NoHashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.055
2016-01-14 17:40:18,578 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] executed rule registry: HashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.074
2016-01-14 17:40:18,578 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: NoHashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:18,578 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: HashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:18,579 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: NoHashJoinHadoop2TezRuleRegistry, result was selected using: 'default comparator: selects plan with fewest steps and fewest nodes'
2016-01-14 17:40:18,600 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt
2016-01-14 17:40:18,626 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistance/euclidean/long
2016-01-14 17:40:18,780 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] starting
2016-01-14 17:40:18,781 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:18,781 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  sink: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistance/euclidean/long"]
2016-01-14 17:40:18,782 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  parallel execution of steps is enabled: false
2016-01-14 17:40:18,782 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  executing total steps: 1
2016-01-14 17:40:18,782 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  allocating management threads: 1
2016-01-14 17:40:18,783 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] starting step: (1/1) ...anDistance/euclidean/long
2016-01-14 17:40:18,783 WARN  flow.Flow (BaseFlow.java:logWarn(1445)) - [euclidean] 'yarn.timeline-service.enabled' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon
2016-01-14 17:40:18,802 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tez session mode enabled: true
2016-01-14 17:40:18,806 INFO  client.TezClient (TezClient.java:<init>(153)) - Tez Client Version: [ component=tez-api, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:18,807 INFO  client.TezClient (TezClient.java:start(333)) - Session mode. Starting session.
2016-01-14 17:40:18,807 INFO  client.TezClientUtils (TezClientUtils.java:setupTezJarsLocalResources(159)) - Ignoring 'tez.lib.uris' since  'tez.ignore.lib.uris' is set to true
2016-01-14 17:40:18,816 INFO  client.TezClient (TezCommonUtils.java:createTezSystemStagingPath(118)) - Tez system stage directory file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001 doesn't exist and is created
2016-01-14 17:40:18,833 INFO  client.LocalClient (LocalClient.java:createDAGAppMaster(328)) - DAGAppMaster thread has been created
2016-01-14 17:40:18,833 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:18,834 INFO  client.LocalClient (LocalClient.java:run(275)) - Using staging directory: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001
2016-01-14 17:40:18,836 INFO  app.DAGAppMaster (DAGAppMaster.java:<init>(309)) - Created DAGAppMaster for application appattempt_1452793218807_0001_000000, versionInfo=[ component=tez-dag, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:18,854 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(333)) - Comparing client version with AM version, clientVersion=0.6.2, AMVersion=0.6.2
2016-01-14 17:40:18,855 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(386)) - Adding session token to jobTokenSecretManager for application
2016-01-14 17:40:18,857 INFO  node.AMNodeTracker (AMNodeTracker.java:serviceInit(77)) - blacklistDisablePercent is 33, blacklistingEnabled: false, maxTaskFailuresPerNode: 1
2016-01-14 17:40:18,858 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceInit(50)) - Initializing HistoryEventHandler
2016-01-14 17:40:18,858 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(66)) - Log file location for SimpleHistoryLoggingService not specified, defaulting to containerLogDir=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001/localmode-log-dir
2016-01-14 17:40:18,858 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(88)) - Initializing SimpleHistoryLoggingService, logFileLocation=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001/localmode-log-dir/history.txt.appattempt_1452793218807_0001_000000, maxErrors=10
2016-01-14 17:40:18,858 INFO  recovery.RecoveryService (RecoveryService.java:serviceInit(109)) - Initializing RecoveryService
2016-01-14 17:40:18,859 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_LAUNCHED]: appAttemptId=appattempt_1452793218807_0001_000000, appSubmitTime=1452793218835, launchTime=1452793218835
2016-01-14 17:40:18,866 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStart(73)) - Starting HistoryEventHandler
2016-01-14 17:40:18,868 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStart(95)) - Starting SimpleHistoryLoggingService
2016-01-14 17:40:18,869 INFO  recovery.RecoveryService (RecoveryService.java:serviceStart(127)) - Starting RecoveryService
2016-01-14 17:40:18,869 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_LAUNCHED to history file
2016-01-14 17:40:18,870 INFO  client.DAGClientServer (DAGClientServer.java:serviceStart(91)) - Instantiated DAGClientRPCServer at ip-10-187-41-40.ec2.internal/10.187.41.40:53496
2016-01-14 17:40:18,874 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-dagappmaster.properties,hadoop-metrics2.properties
2016-01-14 17:40:18,877 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_STARTED to history file
2016-01-14 17:40:18,877 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_STARTED]: appAttemptId=appattempt_1452793218807_0001_000000, startTime=1452793218877
2016-01-14 17:40:18,877 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceStart(1644)) - In Session mode. Waiting for DAG over RPC
2016-01-14 17:40:18,934 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: IDLE
2016-01-14 17:40:18,934 INFO  client.TezClient (TezClient.java:start(367)) - The url to track the Tez Session: N/A
2016-01-14 17:40:18,934 INFO  client.TezClient (TezClient.java:submitDAGSession(405)) - Submitting dag to TezSession, sessionName=(1/1) ...anDistance/euclidean/long, applicationId=application_1452793218807_0001, dagName=[A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long
2016-01-14 17:40:18,966 INFO  app.DAGAppMaster (DAGAppMaster.java:submitDAGToAppMaster(1126)) - Starting DAG submitted via RPC: [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long
2016-01-14 17:40:18,969 INFO  app.DAGAppMaster (DAGAppMaster.java:generateDAGVizFile(870)) - Generating DAG graphviz file, dagId=dag_1452793218807_0001_1, filePath=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001/localmode-log-dir/dag_1452793218807_0001_1.dot
2016-01-14 17:40:18,971 INFO  app.DAGAppMaster (DAGAppMaster.java:writePBTextFile(887)) - Writing DAG plan to: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001/localmode-log-dir/dag_1452793218807_0001_1-tez-dag.pb.txt
2016-01-14 17:40:19,021 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793218807_0001_1
2016-01-14 17:40:19,022 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,022 INFO  app.DAGAppMaster (DAGAppMaster.java:startDAG(2051)) - Running DAG: [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long
2016-01-14 17:40:19 Running Dag: dag_1452793218807_0001_1
2016-01-14 17:40:19,036 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_SUBMITTED to history file
2016-01-14 17:40:19,037 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:DAG_SUBMITTED]: dagID=dag_1452793218807_0001_1, submitTime=1452793218966
2016-01-14 17:40:19,038 INFO  impl.VertexImpl (VertexImpl.java:setAdditionalOutputs(4033)) - setting additional outputs for vertex A27E7D0D1CEB47FB839D6908F82F9886
2016-01-14 17:40:19,039 INFO  impl.DAGImpl (DAGImpl.java:assignDAGScheduler(1381)) - Using DAG Scheduler: org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
2016-01-14 17:40:19,040 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_INITIALIZED to history file
2016-01-14 17:40:19,040 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:DAG_INITIALIZED]: dagID=dag_1452793218807_0001_1, initTime=1452793219037
2016-01-14 17:40:19,040 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793218807_0001_1 transitioned from NEW to INITED due to event DAG_INIT
2016-01-14 17:40:19,041 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:DAG_STARTED]: dagID=dag_1452793218807_0001_1, startTime=1452793219041
2016-01-14 17:40:19,041 INFO  impl.DAGImpl (DAGImpl.java:transition(1614)) - Added additional resources : [[]] to classpath
2016-01-14 17:40:19,041 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_STARTED to history file
2016-01-14 17:40:19,042 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793218807_0001_1 transitioned from INITED to RUNNING due to event DAG_START
2016-01-14 17:40:19,042 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2068)) - Root Inputs exist for Vertex: E09EC5EDD6664EF0B1126A2D04BDA5A2 : {636C9D0891BC4939921A39769D3226AE={InputName=636C9D0891BC4939921A39769D3226AE}, {Descriptor=ClassName=org.apache.tez.mapreduce.input.MRInput, hasPayload=true}, {ControllerDescriptor=ClassName=org.apache.tez.mapreduce.common.MRInputAMSplitGenerator, hasPayload=false}}
2016-01-14 17:40:19,042 INFO  client.TezClient (TezClient.java:submitDAGSession(478)) - Submitted dag to TezSession, sessionName=(1/1) ...anDistance/euclidean/long, applicationId=application_1452793218807_0001, dagName=[A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long
2016-01-14 17:40:19,042 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2078)) - Starting root input initializer for input: 636C9D0891BC4939921A39769D3226AE, with class: [org.apache.tez.mapreduce.common.MRInputAMSplitGenerator]
2016-01-14 17:40:19,043 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2191)) - Setting vertexManager to RootInputVertexManager for vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,043 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2983)) - Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split to set #tasks for the vertex vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,043 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2987)) - Vertex will initialize from input initializer. vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,044 INFO  impl.VertexImpl (VertexImpl.java:setupInputInitializerManager(3933)) - Vertex will initialize via inputInitializers vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]. Starting root input initializers: 1
2016-01-14 17:40:19,044 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] transitioned from NEW to INITIALIZING due to event V_INIT
2016-01-14 17:40:19,044 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,044 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:run(242)) - Starting InputInitializer for Input: 636C9D0891BC4939921A39769D3226AE on vertex vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,060 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(92)) - Emitting serialized splits: true
2016-01-14 17:40:19,061 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(110)) - Input 636C9D0891BC4939921A39769D3226AE asking for 1 tasks. Headroom: 672 Task Resource: 1024 waves: 1.7
2016-01-14 17:40:19,061 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(121)) - Grouping input splits
2016-01-14 17:40:19,062 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(294)) - Generating mapred api input splits
2016-01-14 17:40:19,063 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:19,063 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,064 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,064 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(300)) - NumSplits: 1, SerializedSize: 404
2016-01-14 17:40:19,064 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:19,065 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:onSuccess(272)) - Succeeded InputInitializer for Input: 636C9D0891BC4939921A39769D3226AE on vertex vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,065 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_INITIALIZED]: vertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, vertexId=vertex_1452793218807_0001_1_01, initRequestedTime=1452793219044, initedTime=1452793219064, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:19,065 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:19,066 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] numTasks: -1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:19,066 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1374)) - Got updated RootInputsSpecs: {636C9D0891BC4939921A39769D3226AE=forAllWorkUnits=true, update=[1]}
2016-01-14 17:40:19,066 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1393)) - Vertex vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] parallelism set to 1
2016-01-14 17:40:19,066 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] numTasks: 1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:19,067 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793218807_0001_1_00, eventCount=1
2016-01-14 17:40:19,070 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:19,070 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_INITIALIZED]: vertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, vertexId=vertex_1452793218807_0001_1_00, initRequestedTime=1452793219042, initedTime=1452793219067, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=1
2016-01-14 17:40:19,070 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] with distanceFromRoot: 0
2016-01-14 17:40:19,070 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] transitioned from INITIALIZING to INITED due to event V_ROOT_INPUT_INITIALIZED
2016-01-14 17:40:19,070 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,075 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:19,075 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,076 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,076 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:19,076 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_INITIALIZED]: vertexName=A3E8C9F4F0DE42E5A68170940BBC0367, vertexId=vertex_1452793218807_0001_1_02, initRequestedTime=1452793219070, initedTime=1452793219076, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:19,076 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:19,077 INFO  impl.ImmediateStartVertexManager (ImmediateStartVertexManager.java:scheduleTasks(99)) - Starting 1 in E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,078 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,078 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:19,078 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793218807_0001_1_00, startRequestedTime=1452793219077, startedTime=1452793219077
2016-01-14 17:40:19,079 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:19,079 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,082 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:19,082 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,082 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,082 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1915)) - Invoking committer inits for vertex, vertexId=vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,082 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1927)) - Instantiating committer for output=58BC36057BA04E3C9EB867E7AD3C1909, vertexId=vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886], committerClass=org.apache.tez.mapreduce.committer.MROutputCommitter
2016-01-14 17:40:19,082 INFO  impl.VertexImpl (VertexImpl.java:run(1945)) - Invoking committer init for output=58BC36057BA04E3C9EB867E7AD3C1909, vertexId=vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,085 INFO  committer.MROutputCommitter (MROutputCommitter.java:getOutputCommitter(145)) - OutputCommitter for outputName=58BC36057BA04E3C9EB867E7AD3C1909, vertexName=A27E7D0D1CEB47FB839D6908F82F9886, outputCommitterClass=org.apache.hadoop.mapred.FileOutputCommitter
2016-01-14 17:40:19,085 INFO  impl.VertexImpl (VertexImpl.java:run(1949)) - Invoking committer setup for output=58BC36057BA04E3C9EB867E7AD3C1909, vertexId=vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,086 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] submitted tez dag to app master: application_1452793218807_0001, with dag id: dag_1452793218807_0001_1
2016-01-14 17:40:19,087 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:19,088 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_INITIALIZED]: vertexName=A27E7D0D1CEB47FB839D6908F82F9886, vertexId=vertex_1452793218807_0001_1_03, initRequestedTime=1452793219079, initedTime=1452793219087, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:19,088 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:19,088 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:19,089 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_STARTED]: vertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, taskId=task_1452793218807_0001_1_00_000000, scheduledTime=1452793219088, launchTime=1452793219088
2016-01-14 17:40:19,089 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_00_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:19,090 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793218807_0001_1_00 for vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] numStartedSources: 1 numSources: 1
2016-01-14 17:40:19,090 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] with distanceFromRoot: 1
2016-01-14 17:40:19,090 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,093 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: E09EC5EDD6664EF0B1126A2D04BDA5A2 in vertex: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,093 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: CE7D7ECE0B7D4798830D1CC2A9FA27C5 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:19,093 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793218807_0001_1_01, startRequestedTime=1452793219090, startedTime=1452793219090
2016-01-14 17:40:19,093 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:19,093 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:19,094 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: E09EC5EDD6664EF0B1126A2D04BDA5A2, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=636C9D0891BC4939921A39769D3226AE, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:19,094 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_00_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:19,095 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793218807_0001_1_01 for vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] numStartedSources: 1 numSources: 1
2016-01-14 17:40:19,095 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] with distanceFromRoot: 2
2016-01-14 17:40:19,095 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:19,095 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: CE7D7ECE0B7D4798830D1CC2A9FA27C5 in vertex: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,096 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: A3E8C9F4F0DE42E5A68170940BBC0367 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:19,096 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:19,096 INFO  node.AMNodeTracker (AMNodeTracker.java:nodeSeen(91)) - Adding new node: 127.0.0.1:0
2016-01-14 17:40:19,096 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793218807_0001_1_02, startRequestedTime=1452793219095, startedTime=1452793219095
2016-01-14 17:40:19,098 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:19,098 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(112)) - Adding #1 tokens and #0 secret keys for NM use for launching container
2016-01-14 17:40:19,099 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(122)) - Putting shuffle token in serviceData
2016-01-14 17:40:19,100 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793218807_0001_00_000001
2016-01-14 17:40:19,100 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:19,100 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793218807_0001_1_02 for vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] numStartedSources: 1 numSources: 1
2016-01-14 17:40:19,100 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] with distanceFromRoot: 3
2016-01-14 17:40:19,101 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: A3E8C9F4F0DE42E5A68170940BBC0367 in vertex: A27E7D0D1CEB47FB839D6908F82F9886
2016-01-14 17:40:19,101 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: A27E7D0D1CEB47FB839D6908F82F9886 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:19,102 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:19,102 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793218807_0001_1_03, startRequestedTime=1452793219101, startedTime=1452793219101
2016-01-14 17:40:19,102 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:19,103 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793218807_0001_00_000001
2016-01-14 17:40:19,103 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - DAGAppMaster metrics system already initialized!
2016-01-14 17:40:19,104 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:19,104 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793218807_0001_00_000001, launchTime=1452793219104
2016-01-14 17:40:19,104 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:19,105 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000001
2016-01-14 17:40:19,106 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793218807_0001_1_00_000000_0] to container: [container_1452793218807_0001_00_000001]
2016-01-14 17:40:19,106 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:19,106 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793218807_0001_00_000001 given task: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,106 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000001: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,106 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,106 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793218807_0001_1_00_000000_0] started. Is using containerId: [container_1452793218807_0001_00_000001] on NM: [127.0.0.1:0]
2016-01-14 17:40:19,107 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,107 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:19,107 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:19,107 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:19,107 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, taskAttemptId=attempt_1452793218807_0001_1_00_000000_0, startTime=1452793219106, containerId=container_1452793218807_0001_00_000001, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793218807_0001_00_000001/teamcity, completedLogs=
2016-01-14 17:40:19,107 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:19,108 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:19,108 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:19,108 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: E09EC5EDD6664EF0B1126A2D04BDA5A2, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=636C9D0891BC4939921A39769D3226AE, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:19,108 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_00_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:19,108 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:19,109 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_00_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:19,109 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,110 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:19,110 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,110 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793218807_0001_1_00_000000_0 sent events: (0-1)
2016-01-14 17:40:19,111 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,111 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:19,111 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=636C9D0891BC4939921A39769D3226AE, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }
2016-01-14 17:40:19,112 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:19,112 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:19,112 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,113 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: INPUT, componentVertexName: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,113 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,116 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:19,123 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:19,123 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,124 INFO  input.MRInput (MRInput.java:initialize(440)) - Using New mapreduce API: false, split information via event: true
2016-01-14 17:40:19,124 INFO  input.MRInput (MRInput.java:initializeInternal(493)) - Initialzed MRInput: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,124 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,125 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,129 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: E09EC5EDD6664EF0B1126A2D04BDA5A2, ordinal: 0
2016-01-14 17:40:19,130 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: E09EC5EDD6664EF0B1126A2D04BDA5A2, mem on start (mb), free: 245, total: 452, max: 672
2016-01-14 17:40:19,130 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,130 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:19,130 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:19,130 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:19,131 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:19,131 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.input.MRInput
2016-01-14 17:40:19,131 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:19,131 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:19,131 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:19,132 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 104857600, TotalRequestedScaled: 9.679163076923077E7, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.15
2016-01-14 17:40:19,132 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, 636C9D0891BC4939921A39769D3226AE, org.apache.tez.mapreduce.input.MRInput: requested=0, allocated=0
2016-01-14 17:40:19,132 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, CE7D7ECE0B7D4798830D1CC2A9FA27C5, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=104857600
2016-01-14 17:40:19,132 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:19,132 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: 636C9D0891BC4939921A39769D3226AE being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:19,133 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:19,133 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:19,133 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,133 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:19,133 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,134 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:19,134 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,134 INFO  input.MRInput (MRInput.java:initFromEventInternal(619)) - Initializing RecordReader from event
2016-01-14 17:40:19,135 INFO  input.MRInput (MRInput.java:initFromEventInternal(630)) - Split Details -> SplitClass: org.apache.hadoop.mapred.split.TezGroupedSplit, OldSplit: org.apache.hadoop.mapred.split.TezGroupedSplit@d81c776
2016-01-14 17:40:19,138 INFO  lib.MRReaderMapred (MRReaderMapred.java:setIncrementalConfigParams(163)) - Processing split: org.apache.hadoop.mapred.split.TezGroupedSplit@d81c776
2016-01-14 17:40:19,139 INFO  input.MRInput (MRInput.java:initFromEventInternal(634)) - Initialized RecordReader from event
2016-01-14 17:40:19,139 INFO  input.MRInput (MRInput.java:processSplitEvent(586)) - Notifying on RecordReader Initialized
2016-01-14 17:40:19,141 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:19,143 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"] streamed: true, id: 636C9D0891BC4939921A39769D3226AE
2016-01-14 17:40:19,143 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]], id: 0D4032521C614198B13164AD322CF6DE
2016-01-14 17:40:19,143 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]] 0D4032521C614198B13164AD322CF6DE
2016-01-14 17:40:19,144 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@3185d334; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@48ed1452; comparator=cascading.tuple.hadoop.util.TupleComparator@44d138e5; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:19,145 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:19,145 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:19,146 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:19,160 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 26214396(104857584)
2016-01-14 17:40:19,160 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 100
2016-01-14 17:40:19,160 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 83886080
2016-01-14 17:40:19,160 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 104857600
2016-01-14 17:40:19,160 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 26214396; length = 6553600
2016-01-14 17:40:19,161 INFO  element.TezSourceStage (TezSourceStage.java:prepare(63)) - calling MRInput#start() on: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:19,161 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:19,162 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: E09EC5EDD6664EF0B1126A2D04BDA5A2, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:19,166 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,166 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: E09EC5EDD6664EF0B1126A2D04BDA5A2, mem on close (mb), free: 143, total: 452, max: 672
2016-01-14 17:40:19,166 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:19,166 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:19,166 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 1676; bufvoid = 104857600
2016-01-14 17:40:19,167 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 26214396(104857584); kvend = 26214260(104857040); length = 137/6553600
2016-01-14 17:40:19,170 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,174 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793218807_0001_1_00_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:19,174 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793218807_0001_1_00, eventCount=1
2016-01-14 17:40:19,174 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 27 [[File System Counters FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0][cascading.flow.SliceCounters Process_Begin_Time=1452793219162, Process_Duration=4, Process_End_Time=1452793219166, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2][cascading.flow.StepCounters Tuples_Read=7]]
2016-01-14 17:40:19,175 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:19,175 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,175 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, taskAttemptId=attempt_1452793218807_0001_1_00_000000_0, startTime=1452793219106, finishTime=1452793219175, timeTaken=69, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219162, Process_Duration=4, Process_End_Time=1452793219166, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:19,176 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000001
2016-01-14 17:40:19,176 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_00_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:19,176 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:19,177 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:19,177 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19,178 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:19,178 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_FINISHED]: vertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, taskId=task_1452793218807_0001_1_00_000000, startTime=1452793219106, finishTime=1452793219178, timeTaken=72, status=SUCCEEDED, successfulAttemptID=attempt_1452793218807_0001_1_00_000000_0, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219162, Process_Duration=4, Process_End_Time=1452793219166, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:19,179 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_00_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:19,179 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] : 1
2016-01-14 17:40:19,179 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,180 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:19,180 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_FINISHED]: vertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, vertexId=vertex_1452793218807_0001_1_00, initRequestedTime=1452793219042, initedTime=1452793219067, startRequestedTime=1452793219077, startedTime=1452793219077, finishTime=1452793219179, timeTaken=102, status=SUCCEEDED, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219162, Process_Duration=4, Process_End_Time=1452793219166, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=2, cascading.flow.StepCounters, Tuples_Read=7, vertexStats=firstTaskStartTime=1452793219106, firstTasksToStart=[ task_1452793218807_0001_1_00_000000 ], lastTaskFinishTime=1452793219175, lastTasksToFinish=[ task_1452793218807_0001_1_00_000000 ], minTaskDuration=69, maxTaskDuration=69, avgTaskDuration=69.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793218807_0001_1_00_000000 ], longestDurationTasks=[ task_1452793218807_0001_1_00_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:19,181 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:19,181 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:19,181 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:19,181 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793218807_0001_00_000001, stoppedTime=1452793219181, exitStatus=0
2016-01-14 17:40:19,181 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:19,182 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] attempt: attempt_1452793218807_0001_1_00_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:19,182 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,182 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,182 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2] completed., numCompletedVertices=1, numSuccessfulVertices=1, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:19,183 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793218807_0001_00_000001. Relying on regular task shutdown for it to end
2016-01-14 17:40:19,183 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_STARTED]: vertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, taskId=task_1452793218807_0001_1_01_000000, scheduledTime=1452793219183, launchTime=1452793219183
2016-01-14 17:40:19,183 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:19,183 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_01_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:19,183 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,184 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:19,184 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: CE7D7ECE0B7D4798830D1CC2A9FA27C5, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A3E8C9F4F0DE42E5A68170940BBC0367, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:19,184 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_01_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:19,185 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:19,186 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793218807_0001_00_000002
2016-01-14 17:40:19,186 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:19,187 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793218807_0001_00_000001 is valid, but no longer registered, and will be killed
2016-01-14 17:40:19,187 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000001: 10 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:19,187 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793218807_0001_00_000001, Exiting
2016-01-14 17:40:19,188 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793218807_0001_00_000001
2016-01-14 17:40:19,188 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793218807_0001_00_000001
2016-01-14 17:40:19,190 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793218807_0001_00_000001 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:19,190 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793218807_0001_00_000002
2016-01-14 17:40:19,190 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793218807_0001_00_000001 completed successfully
2016-01-14 17:40:19,191 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000001 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:19,191 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:19,193 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:19,194 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:19,194 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793218807_0001_00_000002, launchTime=1452793219193
2016-01-14 17:40:19,194 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000002
2016-01-14 17:40:19,195 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793218807_0001_1_01_000000_0] to container: [container_1452793218807_0001_00_000002]
2016-01-14 17:40:19,195 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:19,195 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793218807_0001_00_000002 given task: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,196 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000002: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,196 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793218807_0001_1_01_000000_0] started. Is using containerId: [container_1452793218807_0001_00_000002] on NM: [127.0.0.1:0]
2016-01-14 17:40:19,196 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:19,196 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, taskAttemptId=attempt_1452793218807_0001_1_01_000000_0, startTime=1452793219196, containerId=container_1452793218807_0001_00_000002, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793218807_0001_00_000002/teamcity, completedLogs=
2016-01-14 17:40:19,196 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_01_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,197 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_01_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:19,197 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,197 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,197 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:19,198 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:19,198 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:19,198 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:19,198 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:19,198 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: CE7D7ECE0B7D4798830D1CC2A9FA27C5, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A3E8C9F4F0DE42E5A68170940BBC0367, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:19,199 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:19,199 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,201 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:19,201 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,201 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793218807_0001_1_01_000000_0 sent events: (0-1)
2016-01-14 17:40:19,202 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=E09EC5EDD6664EF0B1126A2D04BDA5A2, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:19,202 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,202 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=A3E8C9F4F0DE42E5A68170940BBC0367, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:19,203 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:19,204 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,206 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:19,206 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,206 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,203 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:19,207 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:19,207 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,208 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:19,209 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:19,209 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,209 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,216 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: CE7D7ECE0B7D4798830D1CC2A9FA27C5, ordinal: 1
2016-01-14 17:40:19,216 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: CE7D7ECE0B7D4798830D1CC2A9FA27C5, mem on start (mb), free: 137, total: 452, max: 672
2016-01-14 17:40:19,216 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,216 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:19,216 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:19,216 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:19,216 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:19,217 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:19,217 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:19,217 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:19,217 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:19,217 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, A3E8C9F4F0DE42E5A68170940BBC0367, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:19,217 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, E09EC5EDD6664EF0B1126A2D04BDA5A2, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:19,218 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:19,218 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: E09EC5EDD6664EF0B1126A2D04BDA5A2 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:19,219 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:19,220 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:19,218 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,224 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:19,224 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:19,225 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: E09EC5EDD6664EF0B1126A2D04BDA5A2 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:19,225 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:19,225 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:19,226 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:19,229 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,230 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:19,230 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,231 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793218807_0001_1_00_000000_0_10027, runDuration: 58697, ]
2016-01-14 17:40:19,234 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]]
2016-01-14 17:40:19,234 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793218807_0001_1_00_000000_0_10027] to DISK_DIRECT, CompressedSize=1089, DecompressedSize=1085,EndTime=1452793219234, TimeTaken=2, Rate=0.52 MB/s
2016-01-14 17:40:19,234 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,234 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:19,234 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [E09EC5EDD6664EF0B1126A2D04BDA5A2] #7 in 2ms
2016-01-14 17:40:19,235 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]] streamed: true, id: 0D4032521C614198B13164AD322CF6DE
2016-01-14 17:40:19,235 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [E09EC5EDD6664EF0B1126A2D04BDA5A2] #7, status:false, isInterrupted:false
2016-01-14 17:40:19,235 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(euclidean)[by:[{?}:ALL]], id: 09E8E91AABAA4A468BA2F7220CCC53A5
2016-01-14 17:40:19,239 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:19,239 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(euclidean)[by:[{?}:ALL]] 09E8E91AABAA4A468BA2F7220CCC53A5
2016-01-14 17:40:19,239 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 1089 bytes from disk
2016-01-14 17:40:19,240 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:19,240 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:19,240 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@e06275c; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@36021035; comparator=cascading.tuple.hadoop.util.TupleComparator@41762346; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:19,240 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 1061 bytes
2016-01-14 17:40:19,241 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,241 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:19,241 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:19,242 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:19,241 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:19,251 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:19,252 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:19,252 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:19,252 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:19,252 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:19,253 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]] 0D4032521C614198B13164AD322CF6DE, for 1 inputs
2016-01-14 17:40:19,254 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(89)) - attempting to load codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:19,254 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(94)) - found codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:19,254 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: CE7D7ECE0B7D4798830D1CC2A9FA27C5, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:19,255 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@7e4ed377; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@691d6a69
2016-01-14 17:40:19,282 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,282 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: CE7D7ECE0B7D4798830D1CC2A9FA27C5, mem on close (mb), free: 70, total: 452, max: 672
2016-01-14 17:40:19,283 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: E09EC5EDD6664EF0B1126A2D04BDA5A2
2016-01-14 17:40:19,283 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:19,283 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:19,283 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:19,283 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:19,283 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:19,283 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:19,284 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:19,284 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 13104; bufvoid = 66060288
2016-01-14 17:40:19,284 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514376(66057504); length = 693/4128768
2016-01-14 17:40:19,294 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,297 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793218807_0001_1_01_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:19,298 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793218807_0001_1_01, eventCount=1
2016-01-14 17:40:19,298 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=16, FIRST_EVENT_RECEIVED=7, LAST_EVENT_RECEIVED=7][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793219254, Process_Duration=28, Process_End_Time=1452793219282, Read_Duration=2, Tuples_Read=35, Tuples_Written=174, Write_Duration=10]]
2016-01-14 17:40:19,299 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:19,299 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, taskAttemptId=attempt_1452793218807_0001_1_01_000000_0, startTime=1452793219196, finishTime=1452793219298, timeTaken=102, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=16, FIRST_EVENT_RECEIVED=7, LAST_EVENT_RECEIVED=7, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219254, Process_Duration=28, Process_End_Time=1452793219282, Read_Duration=2, Tuples_Read=35, Tuples_Written=174, Write_Duration=10
2016-01-14 17:40:19,299 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:19,300 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,299 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_01_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:19,300 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:19,300 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19,301 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:19,301 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_FINISHED]: vertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, taskId=task_1452793218807_0001_1_01_000000, startTime=1452793219196, finishTime=1452793219300, timeTaken=104, status=SUCCEEDED, successfulAttemptID=attempt_1452793218807_0001_1_01_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=16, FIRST_EVENT_RECEIVED=7, LAST_EVENT_RECEIVED=7, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219254, Process_Duration=28, Process_End_Time=1452793219282, Read_Duration=2, Tuples_Read=35, Tuples_Written=174, Write_Duration=10
2016-01-14 17:40:19,301 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_01_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:19,301 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] : 1
2016-01-14 17:40:19,301 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,302 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000002
2016-01-14 17:40:19,303 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_FINISHED]: vertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, vertexId=vertex_1452793218807_0001_1_01, initRequestedTime=1452793219044, initedTime=1452793219064, startRequestedTime=1452793219090, startedTime=1452793219090, finishTime=1452793219302, timeTaken=212, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=16, FIRST_EVENT_RECEIVED=7, LAST_EVENT_RECEIVED=7, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219254, Process_Duration=28, Process_End_Time=1452793219282, Read_Duration=2, Tuples_Read=35, Tuples_Written=174, Write_Duration=10, vertexStats=firstTaskStartTime=1452793219196, firstTasksToStart=[ task_1452793218807_0001_1_01_000000 ], lastTaskFinishTime=1452793219298, lastTasksToFinish=[ task_1452793218807_0001_1_01_000000 ], minTaskDuration=102, maxTaskDuration=102, avgTaskDuration=102.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793218807_0001_1_01_000000 ], longestDurationTasks=[ task_1452793218807_0001_1_01_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:19,303 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:19,303 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:19,304 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:19,305 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793218807_0001_00_000002, stoppedTime=1452793219305, exitStatus=0
2016-01-14 17:40:19,305 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:19,305 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] attempt: attempt_1452793218807_0001_1_01_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:19,305 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,305 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,306 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5] completed., numCompletedVertices=2, numSuccessfulVertices=2, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:19,306 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793218807_0001_00_000002. Relying on regular task shutdown for it to end
2016-01-14 17:40:19,306 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_STARTED]: vertexName=A3E8C9F4F0DE42E5A68170940BBC0367, taskId=task_1452793218807_0001_1_02_000000, scheduledTime=1452793219306, launchTime=1452793219306
2016-01-14 17:40:19,306 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_02_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:19,306 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:19,306 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,307 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:19,307 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: A3E8C9F4F0DE42E5A68170940BBC0367, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A27E7D0D1CEB47FB839D6908F82F9886, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:19,307 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_02_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:19,307 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:19,307 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:19,308 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793218807_0001_00_000003
2016-01-14 17:40:19,308 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:19,310 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793218807_0001_00_000003
2016-01-14 17:40:19,310 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - TezTask metrics system already initialized!
2016-01-14 17:40:19,312 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:19,312 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793218807_0001_00_000003, launchTime=1452793219311
2016-01-14 17:40:19,312 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:19,313 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000003
2016-01-14 17:40:19,313 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793218807_0001_00_000002 is valid, but no longer registered, and will be killed
2016-01-14 17:40:19,313 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000002: 11 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:19,314 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793218807_0001_00_000002, Exiting
2016-01-14 17:40:19,314 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793218807_0001_00_000002
2016-01-14 17:40:19,314 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793218807_0001_00_000002
2016-01-14 17:40:19,315 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793218807_0001_00_000002 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:19,316 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793218807_0001_00_000002 completed successfully
2016-01-14 17:40:19,316 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000002 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:19,316 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793218807_0001_1_02_000000_0] to container: [container_1452793218807_0001_00_000003]
2016-01-14 17:40:19,317 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:19,317 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793218807_0001_00_000003 given task: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,318 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793218807_0001_1_02_000000_0] started. Is using containerId: [container_1452793218807_0001_00_000003] on NM: [127.0.0.1:0]
2016-01-14 17:40:19,318 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000003: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,318 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,318 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=A3E8C9F4F0DE42E5A68170940BBC0367, taskAttemptId=attempt_1452793218807_0001_1_02_000000_0, startTime=1452793219317, containerId=container_1452793218807_0001_00_000003, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793218807_0001_00_000003/teamcity, completedLogs=
2016-01-14 17:40:19,319 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,319 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_02_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:19,319 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,319 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_02_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:19,319 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:19,319 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:19,319 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:19,319 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:19,319 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:19,319 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: A3E8C9F4F0DE42E5A68170940BBC0367, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=A27E7D0D1CEB47FB839D6908F82F9886, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:19,320 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:19,323 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,323 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:19,325 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,323 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793218807_0001_1_02_000000_0 sent events: (0-1)
2016-01-14 17:40:19,326 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,328 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=A27E7D0D1CEB47FB839D6908F82F9886, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:19,329 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:19,330 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: A27E7D0D1CEB47FB839D6908F82F9886
2016-01-14 17:40:19,331 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=CE7D7ECE0B7D4798830D1CC2A9FA27C5, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:19,331 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:19,332 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:19,332 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,333 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:19,334 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: A27E7D0D1CEB47FB839D6908F82F9886
2016-01-14 17:40:19,334 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: A27E7D0D1CEB47FB839D6908F82F9886
2016-01-14 17:40:19,337 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:19,337 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:19,337 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,340 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,347 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: A3E8C9F4F0DE42E5A68170940BBC0367, ordinal: 2
2016-01-14 17:40:19,347 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A3E8C9F4F0DE42E5A68170940BBC0367, mem on start (mb), free: 63, total: 452, max: 672
2016-01-14 17:40:19,347 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,347 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:19,347 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:19,347 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:19,347 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:19,348 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:19,348 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:19,348 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:19,348 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:19,349 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, A27E7D0D1CEB47FB839D6908F82F9886, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:19,349 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, CE7D7ECE0B7D4798830D1CC2A9FA27C5, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:19,349 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:19,349 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: CE7D7ECE0B7D4798830D1CC2A9FA27C5 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:19,349 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,351 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:19,351 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:19,351 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:19,352 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:19,352 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: CE7D7ECE0B7D4798830D1CC2A9FA27C5 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:19,353 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:19,353 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:19,353 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:19,357 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,361 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:19,361 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,361 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793218807_0001_1_01_000000_0_10030, runDuration: 91377, ]
2016-01-14 17:40:19,364 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793218807_0001_1_01_000000_0_10030] to DISK_DIRECT, CompressedSize=7519, DecompressedSize=7515,EndTime=1452793219364, TimeTaken=1, Rate=7.17 MB/s
2016-01-14 17:40:19,365 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,366 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(euclidean)[by:[{?}:ALL]]
2016-01-14 17:40:19,366 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.01 MB/s)
2016-01-14 17:40:19,367 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [CE7D7ECE0B7D4798830D1CC2A9FA27C5] #8 in 4ms
2016-01-14 17:40:19,367 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(euclidean)[by:[{?}:ALL]] streamed: true, id: 09E8E91AABAA4A468BA2F7220CCC53A5
2016-01-14 17:40:19,368 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [CE7D7ECE0B7D4798830D1CC2A9FA27C5] #8, status:false, isInterrupted:false
2016-01-14 17:40:19,369 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:19,369 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 7519 bytes from disk
2016-01-14 17:40:19,370 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:19,370 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:19,370 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 7443 bytes
2016-01-14 17:40:19,370 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,371 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:19,371 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(euclidean)[by:[{2}:'name1', 'name2']], id: 1C551073D714416EB8D0CBA0EB2A7C6E
2016-01-14 17:40:19,371 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(euclidean)[by:[{2}:'name1', 'name2']] 1C551073D714416EB8D0CBA0EB2A7C6E
2016-01-14 17:40:19,372 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@48458948; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@4d37fb09; comparator=cascading.tuple.hadoop.util.TupleComparator@1642337a; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:19,372 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:19,372 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:19,372 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:19,393 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:19,393 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:19,393 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:19,393 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:19,393 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:19,394 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(euclidean)[by:[{?}:ALL]] 09E8E91AABAA4A468BA2F7220CCC53A5, for 1 inputs
2016-01-14 17:40:19,394 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: A3E8C9F4F0DE42E5A68170940BBC0367, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:19,395 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@bcabcf; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@62b3a369
2016-01-14 17:40:19,417 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,417 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A3E8C9F4F0DE42E5A68170940BBC0367, mem on close (mb), free: 198, total: 453, max: 672
2016-01-14 17:40:19,417 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: CE7D7ECE0B7D4798830D1CC2A9FA27C5
2016-01-14 17:40:19,417 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:19,418 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:19,418 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:19,418 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:19,418 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:19,419 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:19,419 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:19,420 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 7074; bufvoid = 66060288
2016-01-14 17:40:19,420 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514724(66058896); length = 345/4128768
2016-01-14 17:40:19,425 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,430 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793218807_0001_1_02_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:19,431 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793218807_0001_1_02, eventCount=1
2016-01-14 17:40:19,431 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:19,432 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5159, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=9, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=16, MERGE_PHASE_TIME=18, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793219394, Process_Duration=23, Process_End_Time=1452793219417, Read_Duration=7, Tuples_Read=174, Tuples_Written=87, Write_Duration=3]]
2016-01-14 17:40:19,433 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:19,433 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=A3E8C9F4F0DE42E5A68170940BBC0367, taskAttemptId=attempt_1452793218807_0001_1_02_000000_0, startTime=1452793219317, finishTime=1452793219431, timeTaken=114, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5159, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=9, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=16, MERGE_PHASE_TIME=18, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219394, Process_Duration=23, Process_End_Time=1452793219417, Read_Duration=7, Tuples_Read=174, Tuples_Written=87, Write_Duration=3
2016-01-14 17:40:19,433 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,433 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_02_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:19,435 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19,435 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:19,436 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_FINISHED]: vertexName=A3E8C9F4F0DE42E5A68170940BBC0367, taskId=task_1452793218807_0001_1_02_000000, startTime=1452793219317, finishTime=1452793219435, timeTaken=118, status=SUCCEEDED, successfulAttemptID=attempt_1452793218807_0001_1_02_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5159, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=9, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=16, MERGE_PHASE_TIME=18, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219394, Process_Duration=23, Process_End_Time=1452793219417, Read_Duration=7, Tuples_Read=174, Tuples_Written=87, Write_Duration=3
2016-01-14 17:40:19,436 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_02_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:19,436 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] : 1
2016-01-14 17:40:19,436 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,438 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_FINISHED]: vertexName=A3E8C9F4F0DE42E5A68170940BBC0367, vertexId=vertex_1452793218807_0001_1_02, initRequestedTime=1452793219070, initedTime=1452793219076, startRequestedTime=1452793219095, startedTime=1452793219095, finishTime=1452793219436, timeTaken=341, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5159, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=9, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=16, MERGE_PHASE_TIME=18, FIRST_EVENT_RECEIVED=10, LAST_EVENT_RECEIVED=10, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219394, Process_Duration=23, Process_End_Time=1452793219417, Read_Duration=7, Tuples_Read=174, Tuples_Written=87, Write_Duration=3, vertexStats=firstTaskStartTime=1452793219317, firstTasksToStart=[ task_1452793218807_0001_1_02_000000 ], lastTaskFinishTime=1452793219431, lastTasksToFinish=[ task_1452793218807_0001_1_02_000000 ], minTaskDuration=114, maxTaskDuration=114, avgTaskDuration=114.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793218807_0001_1_02_000000 ], longestDurationTasks=[ task_1452793218807_0001_1_02_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:19,438 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:19,439 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:19,440 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:19,440 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:19,441 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793218807_0001_00_000003, stoppedTime=1452793219440, exitStatus=0
2016-01-14 17:40:19,441 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:19,441 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] attempt: attempt_1452793218807_0001_1_02_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:19,441 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:19,441 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: A27E7D0D1CEB47FB839D6908F82F9886
2016-01-14 17:40:19,442 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,434 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000003
2016-01-14 17:40:19,442 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793218807_0001_00_000003 is valid, but no longer registered, and will be killed
2016-01-14 17:40:19,443 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000003: 0 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:19,442 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367] completed., numCompletedVertices=3, numSuccessfulVertices=3, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:19,443 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793218807_0001_00_000003. Relying on regular task shutdown for it to end
2016-01-14 17:40:19,444 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:19,444 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_STARTED]: vertexName=A27E7D0D1CEB47FB839D6908F82F9886, taskId=task_1452793218807_0001_1_03_000000, scheduledTime=1452793219443, launchTime=1452793219443
2016-01-14 17:40:19,444 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_03_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:19,444 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793218807_0001_00_000003, Exiting
2016-01-14 17:40:19,444 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793218807_0001_00_000003
2016-01-14 17:40:19,445 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793218807_0001_00_000003
2016-01-14 17:40:19,446 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793218807_0001_00_000003 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:19,446 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793218807_0001_00_000003 completed successfully
2016-01-14 17:40:19,447 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,447 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:19,447 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000003 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:19,448 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: A27E7D0D1CEB47FB839D6908F82F9886, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=A3E8C9F4F0DE42E5A68170940BBC0367, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=58BC36057BA04E3C9EB867E7AD3C1909, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:19,448 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_03_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:19,448 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:19,449 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793218807_0001_00_000004
2016-01-14 17:40:19,449 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000004 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:19,451 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793218807_0001_00_000004
2016-01-14 17:40:19,451 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:19,454 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793218807_0001_00_000004, launchTime=1452793219454
2016-01-14 17:40:19,454 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:19,454 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000004 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:19,454 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000004
2016-01-14 17:40:19,457 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793218807_0001_1_03_000000_0] to container: [container_1452793218807_0001_00_000004]
2016-01-14 17:40:19,457 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000004 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:19,458 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793218807_0001_00_000004 given task: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,459 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000004: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,458 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793218807_0001_1_03_000000_0] started. Is using containerId: [container_1452793218807_0001_00_000004] on NM: [127.0.0.1:0]
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,459 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,460 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,460 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:19,460 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:19,460 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:19,460 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:19,460 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:19,460 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1] euclidean/(1/1) ...anDistance/euclidean/long, VertexName: A27E7D0D1CEB47FB839D6908F82F9886, VertexParallelism: 1, TaskAttemptID:attempt_1452793218807_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=A3E8C9F4F0DE42E5A68170940BBC0367, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=58BC36057BA04E3C9EB867E7AD3C1909, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:19,461 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:19,461 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:19,462 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=A27E7D0D1CEB47FB839D6908F82F9886, taskAttemptId=attempt_1452793218807_0001_1_03_000000_0, startTime=1452793219458, containerId=container_1452793218807_0001_00_000004, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793218807_0001_00_000004/teamcity, completedLogs=
2016-01-14 17:40:19,462 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_03_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:19,462 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_03_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:19,463 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,463 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:19,463 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,464 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=A3E8C9F4F0DE42E5A68170940BBC0367, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:19,465 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,465 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=58BC36057BA04E3C9EB867E7AD3C1909, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }
2016-01-14 17:40:19,466 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:19,466 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: 58BC36057BA04E3C9EB867E7AD3C1909
2016-01-14 17:40:19,466 INFO  output.MROutput (MROutput.java:initialize(347)) - Initializing Simple Output
2016-01-14 17:40:19,466 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: OUTPUT, componentVertexName: 58BC36057BA04E3C9EB867E7AD3C1909
2016-01-14 17:40:19,469 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:19,470 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793218807_0001_1_03_000000_0 sent events: (0-1)
2016-01-14 17:40:19,470 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:19,470 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,478 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:19,478 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:19,478 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,478 INFO  output.MROutput (MROutput.java:initialize(418)) - Initialized Simple Output, using_new_api: false
2016-01-14 17:40:19,479 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: 58BC36057BA04E3C9EB867E7AD3C1909
2016-01-14 17:40:19,479 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,485 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: A27E7D0D1CEB47FB839D6908F82F9886, ordinal: 3
2016-01-14 17:40:19,485 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A27E7D0D1CEB47FB839D6908F82F9886, mem on start (mb), free: 191, total: 453, max: 672
2016-01-14 17:40:19,486 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:19,486 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:19,486 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:19,486 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:19,487 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:19,487 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.output.MROutput
2016-01-14 17:40:19,487 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:19,488 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:19,488 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:19,488 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 634178752, TotalRequestedScaled: 5.853957710769231E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.90
2016-01-14 17:40:19,488 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, 58BC36057BA04E3C9EB867E7AD3C1909, org.apache.tez.mapreduce.output.MROutput: requested=0, allocated=0
2016-01-14 17:40:19,489 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, A3E8C9F4F0DE42E5A68170940BBC0367, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=472110858
2016-01-14 17:40:19,489 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:19,489 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: A3E8C9F4F0DE42E5A68170940BBC0367 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:19,489 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,490 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:19,490 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:19,490 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:19,491 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:19,491 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: A3E8C9F4F0DE42E5A68170940BBC0367 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:19,491 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=472110858. Updated to: ShuffleMem=472110858, postMergeMem: 0
2016-01-14 17:40:19,491 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=472110858, maxSingleShuffleLimit=118027712, mergeThreshold=424899744, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:19,492 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:19,493 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,493 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:19,494 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,494 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793218807_0001_1_02_000000_0_10032, runDuration: 98422, ]
2016-01-14 17:40:19,497 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793218807_0001_1_02_000000_0_10032] to DISK_DIRECT, CompressedSize=5127, DecompressedSize=5123,EndTime=1452793219497, TimeTaken=1, Rate=4.89 MB/s
2016-01-14 17:40:19,497 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(euclidean)[by:[{2}:'name1', 'name2']]
2016-01-14 17:40:19,497 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,498 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:19,498 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [A3E8C9F4F0DE42E5A68170940BBC0367] #9 in 2ms
2016-01-14 17:40:19,498 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(euclidean)[by:[{2}:'name1', 'name2']] streamed: true, id: 1C551073D714416EB8D0CBA0EB2A7C6E
2016-01-14 17:40:19,498 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [A3E8C9F4F0DE42E5A68170940BBC0367] #9, status:false, isInterrupted:false
2016-01-14 17:40:19,499 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistance/euclidean/long"], id: 58BC36057BA04E3C9EB867E7AD3C1909
2016-01-14 17:40:19,499 INFO  element.TezSinkStage (TezSinkStage.java:prepare(57)) - calling MROutput#start() on: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistance/euclidean/long"]
2016-01-14 17:40:19,499 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(euclidean)[by:[{2}:'name1', 'name2']] 1C551073D714416EB8D0CBA0EB2A7C6E, for 1 inputs
2016-01-14 17:40:19,500 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:19,500 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 5127 bytes from disk
2016-01-14 17:40:19,500 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:19,500 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:19,501 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 5084 bytes
2016-01-14 17:40:19,501 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,501 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:19,502 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: A27E7D0D1CEB47FB839D6908F82F9886, all 1 inputs ready in: 00:00:00.002
2016-01-14 17:40:19,503 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@2e806493; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@6f4b06be
2016-01-14 17:40:19,511 INFO  output.MROutput (MROutput.java:flush(534)) - Flushing Simple Output
2016-01-14 17:40:19,511 INFO  output.MROutput (MROutput.java:flush(544)) - Flushed Simple Output
2016-01-14 17:40:19,512 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:canCommit(353)) - Commit go/no-go request from attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,512 INFO  impl.TaskImpl (TaskImpl.java:canCommit(750)) - attempt_1452793218807_0001_1_03_000000_0 given a go for committing the task output.
2016-01-14 17:40:19,513 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,513 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: A27E7D0D1CEB47FB839D6908F82F9886, mem on close (mb), free: 188, total: 453, max: 672
2016-01-14 17:40:19,513 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: A3E8C9F4F0DE42E5A68170940BBC0367
2016-01-14 17:40:19,513 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:19,513 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:19,513 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:19,514 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:19,514 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,515 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793218807_0001_1_03_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:19,516 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:19,516 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=A27E7D0D1CEB47FB839D6908F82F9886, taskAttemptId=attempt_1452793218807_0001_1_03_000000_0, startTime=1452793219458, finishTime=1452793219516, timeTaken=58, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=1420, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219502, Process_Duration=11, Process_End_Time=1452793219513, Read_Duration=3, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:19,517 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793218807_0001_1_03_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:19,517 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19,517 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:19,518 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:19,518 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:TASK_FINISHED]: vertexName=A27E7D0D1CEB47FB839D6908F82F9886, taskId=task_1452793218807_0001_1_03_000000, startTime=1452793219458, finishTime=1452793219517, timeTaken=59, status=SUCCEEDED, successfulAttemptID=attempt_1452793218807_0001_1_03_000000_0, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=1420, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219502, Process_Duration=11, Process_End_Time=1452793219513, Read_Duration=3, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:19,518 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793218807_0001_1_03_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:19,518 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] : 1
2016-01-14 17:40:19,518 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,519 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:19,520 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:VERTEX_FINISHED]: vertexName=A27E7D0D1CEB47FB839D6908F82F9886, vertexId=vertex_1452793218807_0001_1_03, initRequestedTime=1452793219079, initedTime=1452793219087, startRequestedTime=1452793219101, startedTime=1452793219101, finishTime=1452793219518, timeTaken=417, status=SUCCEEDED, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=1420, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793219502, Process_Duration=11, Process_End_Time=1452793219513, Read_Duration=3, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21, vertexStats=firstTaskStartTime=1452793219458, firstTasksToStart=[ task_1452793218807_0001_1_03_000000 ], lastTaskFinishTime=1452793219516, lastTasksToFinish=[ task_1452793218807_0001_1_03_000000 ], minTaskDuration=58, maxTaskDuration=58, avgTaskDuration=58.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793218807_0001_1_03_000000 ], longestDurationTasks=[ task_1452793218807_0001_1_03_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:19,520 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:19,521 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000004 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:19,521 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793218807_0001_00_000004, stoppedTime=1452793219521, exitStatus=0
2016-01-14 17:40:19,521 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000004 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:19,521 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886] completed., numCompletedVertices=4, numSuccessfulVertices=4, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:19,521 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(831)) - Calling DAG commit/abort for dag: dag_1452793218807_0001_1
2016-01-14 17:40:19,521 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 46 [[File System Counters FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=1420, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=7, MERGE_PHASE_TIME=10, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793219502, Process_Duration=11, Process_End_Time=1452793219513, Read_Duration=3, Tuples_Read=87, Tuples_Written=21, Write_Duration=0][cascading.flow.StepCounters Tuples_Written=21]]
2016-01-14 17:40:19,522 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:19,522 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:19,522 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,522 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:DAG_COMMIT_STARTED]: dagID=dag_1452793218807_0001_1
2016-01-14 17:40:19,522 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793218807_0001_1_01 [CE7D7ECE0B7D4798830D1CC2A9FA27C5]
2016-01-14 17:40:19,523 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793218807_0001_1_00 [E09EC5EDD6664EF0B1126A2D04BDA5A2]
2016-01-14 17:40:19,523 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(892)) - Committing output: 58BC36057BA04E3C9EB867E7AD3C1909 for vertex: vertex_1452793218807_0001_1_03 [A27E7D0D1CEB47FB839D6908F82F9886]
2016-01-14 17:40:19,523 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793218807_0001_00_000004
2016-01-14 17:40:19,523 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793218807_0001_00_000004 is valid, but no longer registered, and will be killed
2016-01-14 17:40:19,524 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793218807_0001_00_000004: 0 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:19,524 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793218807_0001_00_000004, Exiting
2016-01-14 17:40:19,524 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793218807_0001_00_000004
2016-01-14 17:40:19,524 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793218807_0001_00_000004
2016-01-14 17:40:19,525 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793218807_0001_00_000004 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:19,525 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793218807_0001_00_000004 completed successfully
2016-01-14 17:40:19,531 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793218807_0001_1_02 [A3E8C9F4F0DE42E5A68170940BBC0367]
2016-01-14 17:40:19,535 INFO  recovery.RecoveryService (RecoveryService.java:handle(306)) - DAG completed, dagId=dag_1452793218807_0001_1, queueSize=0
2016-01-14 17:40:19,535 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_FINISHED to history file
2016-01-14 17:40:19,536 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793218807_0001_1][Event:DAG_FINISHED]: dagId=dag_1452793218807_0001_1, startTime=1452793219041, finishTime=1452793219531, timeTaken=490, status=SUCCEEDED, diagnostics=, counters=Counters: 55, org.apache.tez.common.counters.DAGCounter, NUM_SUCCEEDED_TASKS=4, TOTAL_LAUNCHED_TASKS=4, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=15169, FILE_BYTES_WRITTEN=15771, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=111, REDUCE_INPUT_RECORDS=296, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=592, NUM_SHUFFLED_INPUTS=3, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=3, GC_TIME_MILLIS=9, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=1900019712, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=317, OUTPUT_BYTES=21854, OUTPUT_BYTES_WITH_OVERHEAD=13723, OUTPUT_BYTES_PHYSICAL=13735, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=13735, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=13735, SHUFFLE_BYTES_DECOMPRESSED=13723, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=13735, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=33, MERGE_PHASE_TIME=44, FIRST_EVENT_RECEIVED=21, LAST_EVENT_RECEIVED=21, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=5811172877312, Process_Duration=66, Process_End_Time=5811172877378, Read_Duration=13, Tuples_Read=303, Tuples_Written=317, Write_Duration=15, cascading.flow.StepCounters, Tuples_Read=7, Tuples_Written=21
2016-01-14 17:40:19,536 INFO  impl.DAGImpl (DAGImpl.java:finished(1180)) - DAG: dag_1452793218807_0001_1 finished with state: SUCCEEDED
2016-01-14 17:40:19,536 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793218807_0001_1 transitioned from RUNNING to SUCCEEDED due to event DAG_VERTEX_COMPLETED
2016-01-14 17:40:19,537 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(242)) - Ignoring stop request for containerId: container_1452793218807_0001_00_000004
2016-01-14 17:40:19,537 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793218807_0001_00_000004 transitioned from STOP_REQUESTED to COMPLETED via event C_COMPLETED
2016-01-14 17:40:19 Completed Dag: dag_1452793218807_0001_1
2016-01-14 17:40:19,538 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(579)) - DAG completed, dagId=dag_1452793218807_0001_1, dagState=SUCCEEDED
2016-01-14 17:40:19,539 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793218807_0001_1_post
2016-01-14 17:40:19,539 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:19,539 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(625)) - Waiting for next DAG to be submitted.
2016-01-14 17:40:19,544 INFO  client.TezClient (TezClient.java:stop(495)) - Shutting down Tez Session, sessionName=(1/1) ...anDistance/euclidean/long, applicationId=application_1452793218807_0001
2016-01-14 17:40:19,547 INFO  client.DAGClientHandler (DAGClientHandler.java:shutdownAM(117)) - Received message to shutdown AM
2016-01-14 17:40:19,547 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:setShouldUnregisterFlag(580)) - TaskScheduler notified that it should unregister from RM
2016-01-14 17:40:19,547 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdownTezAM(1088)) - No current running DAG, shutting down the AM
2016-01-14 17:40:19,547 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(692)) - DAGAppMasterShutdownHandler invoked
2016-01-14 17:40:19,547 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(701)) - Handling DAGAppMaster shutdown
2016-01-14 17:40:19,548 INFO  app.DAGAppMaster (DAGAppMaster.java:run(721)) - Sleeping for 5 seconds before shutting down
2016-01-14 17:40:19,548 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStop(79)) - Stopping HistoryEventHandler
2016-01-14 17:40:19,549 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(189)) - Stopping RecoveryService
2016-01-14 17:40:19,549 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(192)) - Handle the remaining events in queue, queue size=0
2016-01-14 17:40:19,549 INFO  recovery.RecoveryService (RecoveryService.java:run(166)) - EventQueue take interrupted. Returning
2016-01-14 17:40:19,549 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(214)) - Closing Summary Stream
2016-01-14 17:40:19,550 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStop(118)) - Stopping SimpleHistoryLoggingService, eventQueueBacklog=0
2016-01-14 17:40:19,551 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:run(105)) - EventQueue take interrupted. Returning
2016-01-14 17:40:19,557 INFO  app.DAGAppMaster (DAGAppMaster.java:run(1773)) - Completed deletion of tez scratch data dir, path=file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A93B628F78BE4905BCC34F0BDF04B2B3/D10C4EE62EE2489994E5A762BF8849B1/.tez/application_1452793218807_0001
2016-01-14 17:40:19,557 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  completed in: 00:00.777
2016-01-14 17:40:19,643 INFO  cascading.PlatformTestCase (PlatformTestCase.java:tearDown(189)) - copying to local /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output//hadoop2-tez/distanceusecaseplatform/testEuclideanDistance/euclidean/long
2016-01-14 17:40:19,675 INFO  util.Util (Util.java:findMainClass(1327)) - using default application jar, may cause class not found exceptions on the cluster
2016-01-14 17:40:19,675 INFO  planner.Hadoop2TezPlanner (Hadoop2TezPlanner.java:initialize(149)) - using application jar: /opt/concurrent/home/teamcity/.gradle/caches/modules-2/files-2.1/cascading/cascading-hadoop2-tez/3.0.4/561447ee876073bb61fddc52e2ad157f45c8edd/cascading-hadoop2-tez-3.0.4.jar
2016-01-14 17:40:19,749 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] executed rule registry: NoHashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.043
2016-01-14 17:40:19,770 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] executed rule registry: HashJoinHadoop2TezRuleRegistry, completed as: SUCCESS, in: 00:00.065
2016-01-14 17:40:19,771 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: NoHashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:19,771 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: HashJoinHadoop2TezRuleRegistry, supports assembly with steps: 1, nodes: 4
2016-01-14 17:40:19,771 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] rule registry: NoHashJoinHadoop2TezRuleRegistry, result was selected using: 'default comparator: selects plan with fewest steps and fewest nodes'
2016-01-14 17:40:19,792 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt
2016-01-14 17:40:19,813 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tap forcing step to tez local mode: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceShort/euclidean/short
2016-01-14 17:40:19,965 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] starting
2016-01-14 17:40:19,965 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:19,966 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  sink: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceShort/euclidean/short"]
2016-01-14 17:40:19,966 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  parallel execution of steps is enabled: false
2016-01-14 17:40:19,966 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  executing total steps: 1
2016-01-14 17:40:19,966 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  allocating management threads: 1
2016-01-14 17:40:19,967 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] starting step: (1/1) ...anceShort/euclidean/short
2016-01-14 17:40:19,967 WARN  flow.Flow (BaseFlow.java:logWarn(1445)) - [euclidean] 'yarn.timeline-service.enabled' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon
2016-01-14 17:40:19,981 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] tez session mode enabled: true
2016-01-14 17:40:19,985 INFO  client.TezClient (TezClient.java:<init>(153)) - Tez Client Version: [ component=tez-api, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:19,986 INFO  client.TezClient (TezClient.java:start(333)) - Session mode. Starting session.
2016-01-14 17:40:19,986 INFO  client.TezClientUtils (TezClientUtils.java:setupTezJarsLocalResources(159)) - Ignoring 'tez.lib.uris' since  'tez.ignore.lib.uris' is set to true
2016-01-14 17:40:19,996 INFO  client.TezClient (TezCommonUtils.java:createTezSystemStagingPath(118)) - Tez system stage directory file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001 doesn't exist and is created
2016-01-14 17:40:20,011 INFO  client.LocalClient (LocalClient.java:createDAGAppMaster(328)) - DAGAppMaster thread has been created
2016-01-14 17:40:20,011 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(251)) - DAGAppMaster is not created wait for 100ms...
2016-01-14 17:40:20,012 INFO  client.LocalClient (LocalClient.java:run(275)) - Using staging directory: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001
2016-01-14 17:40:20,014 INFO  app.DAGAppMaster (DAGAppMaster.java:<init>(309)) - Created DAGAppMaster for application appattempt_1452793219985_0001_000000, versionInfo=[ component=tez-dag, version=0.6.2, revision=c04283bf91fd96823eec37056d5c8b1a92728265, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20150804-1436 ]
2016-01-14 17:40:20,026 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(333)) - Comparing client version with AM version, clientVersion=0.6.2, AMVersion=0.6.2
2016-01-14 17:40:20,027 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceInit(386)) - Adding session token to jobTokenSecretManager for application
2016-01-14 17:40:20,029 INFO  node.AMNodeTracker (AMNodeTracker.java:serviceInit(77)) - blacklistDisablePercent is 33, blacklistingEnabled: false, maxTaskFailuresPerNode: 1
2016-01-14 17:40:20,029 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceInit(50)) - Initializing HistoryEventHandler
2016-01-14 17:40:20,029 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(66)) - Log file location for SimpleHistoryLoggingService not specified, defaulting to containerLogDir=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001/localmode-log-dir
2016-01-14 17:40:20,030 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceInit(88)) - Initializing SimpleHistoryLoggingService, logFileLocation=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001/localmode-log-dir/history.txt.appattempt_1452793219985_0001_000000, maxErrors=10
2016-01-14 17:40:20,030 INFO  recovery.RecoveryService (RecoveryService.java:serviceInit(109)) - Initializing RecoveryService
2016-01-14 17:40:20,031 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_LAUNCHED]: appAttemptId=appattempt_1452793219985_0001_000000, appSubmitTime=1452793220013, launchTime=1452793220013
2016-01-14 17:40:20,037 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStart(73)) - Starting HistoryEventHandler
2016-01-14 17:40:20,041 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStart(95)) - Starting SimpleHistoryLoggingService
2016-01-14 17:40:20,043 INFO  recovery.RecoveryService (RecoveryService.java:serviceStart(127)) - Starting RecoveryService
2016-01-14 17:40:20,043 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_LAUNCHED to history file
2016-01-14 17:40:20,048 INFO  client.DAGClientServer (DAGClientServer.java:serviceStart(91)) - Instantiated DAGClientRPCServer at ip-10-187-41-40.ec2.internal/10.187.41.40:55212
2016-01-14 17:40:20,049 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-dagappmaster.properties,hadoop-metrics2.properties
2016-01-14 17:40:20,051 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:N/A][Event:AM_STARTED]: appAttemptId=appattempt_1452793219985_0001_000000, startTime=1452793220051
2016-01-14 17:40:20,051 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event AM_STARTED to history file
2016-01-14 17:40:20,051 INFO  app.DAGAppMaster (DAGAppMaster.java:serviceStart(1644)) - In Session mode. Waiting for DAG over RPC
2016-01-14 17:40:20,111 INFO  client.LocalClient (LocalClient.java:startDAGAppMaster(236)) - DAGAppMaster state: IDLE
2016-01-14 17:40:20,111 INFO  client.TezClient (TezClient.java:start(367)) - The url to track the Tez Session: N/A
2016-01-14 17:40:20,112 INFO  client.TezClient (TezClient.java:submitDAGSession(405)) - Submitting dag to TezSession, sessionName=(1/1) ...anceShort/euclidean/short, applicationId=application_1452793219985_0001, dagName=[A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short
2016-01-14 17:40:20,136 INFO  app.DAGAppMaster (DAGAppMaster.java:submitDAGToAppMaster(1126)) - Starting DAG submitted via RPC: [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short
2016-01-14 17:40:20,138 INFO  app.DAGAppMaster (DAGAppMaster.java:generateDAGVizFile(870)) - Generating DAG graphviz file, dagId=dag_1452793219985_0001_1, filePath=/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001/localmode-log-dir/dag_1452793219985_0001_1.dot
2016-01-14 17:40:20,139 INFO  app.DAGAppMaster (DAGAppMaster.java:writePBTextFile(887)) - Writing DAG plan to: /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001/localmode-log-dir/dag_1452793219985_0001_1-tez-dag.pb.txt
2016-01-14 17:40:20,176 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793219985_0001_1
2016-01-14 17:40:20,177 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,177 INFO  app.DAGAppMaster (DAGAppMaster.java:startDAG(2051)) - Running DAG: [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short
2016-01-14 17:40:20 Running Dag: dag_1452793219985_0001_1
2016-01-14 17:40:20,187 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_SUBMITTED to history file
2016-01-14 17:40:20,188 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:DAG_SUBMITTED]: dagID=dag_1452793219985_0001_1, submitTime=1452793220136
2016-01-14 17:40:20,189 INFO  impl.VertexImpl (VertexImpl.java:setAdditionalOutputs(4033)) - setting additional outputs for vertex FDB3C4710CD245169EDFBD5314D6B5DD
2016-01-14 17:40:20,189 INFO  impl.DAGImpl (DAGImpl.java:assignDAGScheduler(1381)) - Using DAG Scheduler: org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
2016-01-14 17:40:20,191 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:DAG_INITIALIZED]: dagID=dag_1452793219985_0001_1, initTime=1452793220188
2016-01-14 17:40:20,191 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_INITIALIZED to history file
2016-01-14 17:40:20,192 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793219985_0001_1 transitioned from NEW to INITED due to event DAG_INIT
2016-01-14 17:40:20,193 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:DAG_STARTED]: dagID=dag_1452793219985_0001_1, startTime=1452793220193
2016-01-14 17:40:20,193 INFO  impl.DAGImpl (DAGImpl.java:transition(1614)) - Added additional resources : [[]] to classpath
2016-01-14 17:40:20,193 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793219985_0001_1 transitioned from INITED to RUNNING due to event DAG_START
2016-01-14 17:40:20,193 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2068)) - Root Inputs exist for Vertex: F7D6C6EDE1324681B48381DE1AA02837 : {0DA26E84B8D2442A8DA9F5E16634017A={InputName=0DA26E84B8D2442A8DA9F5E16634017A}, {Descriptor=ClassName=org.apache.tez.mapreduce.input.MRInput, hasPayload=true}, {ControllerDescriptor=ClassName=org.apache.tez.mapreduce.common.MRInputAMSplitGenerator, hasPayload=false}}
2016-01-14 17:40:20,194 INFO  impl.VertexImpl (VertexImpl.java:setupVertex(2078)) - Starting root input initializer for input: 0DA26E84B8D2442A8DA9F5E16634017A, with class: [org.apache.tez.mapreduce.common.MRInputAMSplitGenerator]
2016-01-14 17:40:20,194 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2191)) - Setting vertexManager to RootInputVertexManager for vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,194 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2983)) - Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split to set #tasks for the vertex vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,194 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(2987)) - Vertex will initialize from input initializer. vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,193 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_STARTED to history file
2016-01-14 17:40:20,195 INFO  impl.VertexImpl (VertexImpl.java:setupInputInitializerManager(3933)) - Vertex will initialize via inputInitializers vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]. Starting root input initializers: 1
2016-01-14 17:40:20,195 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] transitioned from NEW to INITIALIZING due to event V_INIT
2016-01-14 17:40:20,195 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,195 INFO  client.TezClient (TezClient.java:submitDAGSession(478)) - Submitted dag to TezSession, sessionName=(1/1) ...anceShort/euclidean/short, applicationId=application_1452793219985_0001, dagName=[A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short
2016-01-14 17:40:20,196 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:run(242)) - Starting InputInitializer for Input: 0DA26E84B8D2442A8DA9F5E16634017A on vertex vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,198 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(92)) - Emitting serialized splits: true
2016-01-14 17:40:20,199 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(110)) - Input 0DA26E84B8D2442A8DA9F5E16634017A asking for 1 tasks. Headroom: 672 Task Resource: 1024 waves: 1.7
2016-01-14 17:40:20,200 INFO  common.MRInputAMSplitGenerator (MRInputAMSplitGenerator.java:initialize(121)) - Grouping input splits
2016-01-14 17:40:20,200 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:20,201 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,201 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,201 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:20,201 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(294)) - Generating mapred api input splits
2016-01-14 17:40:20,202 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_INITIALIZED]: vertexName=B82115378D004563BEC7B1F7370537D3, vertexId=vertex_1452793219985_0001_1_01, initRequestedTime=1452793220195, initedTime=1452793220201, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:20,202 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:20,203 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,205 INFO  hadoop.MRInputHelpers (MRInputHelpers.java:generateInputSplitsToMem(300)) - NumSplits: 1, SerializedSize: 404
2016-01-14 17:40:20,207 INFO  dag.RootInputInitializerManager (RootInputInitializerManager.java:onSuccess(272)) - Succeeded InputInitializer for Input: 0DA26E84B8D2442A8DA9F5E16634017A on vertex vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,218 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:20,218 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,219 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,219 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:20,219 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_INITIALIZED]: vertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, vertexId=vertex_1452793219985_0001_1_02, initRequestedTime=1452793220203, initedTime=1452793220219, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:20,220 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:20,220 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] numTasks: -1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:20,220 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1374)) - Got updated RootInputsSpecs: {0DA26E84B8D2442A8DA9F5E16634017A=forAllWorkUnits=true, update=[1]}
2016-01-14 17:40:20,221 INFO  impl.VertexImpl (VertexImpl.java:setParallelism(1393)) - Vertex vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] parallelism set to 1
2016-01-14 17:40:20,221 INFO  impl.VertexImpl (VertexImpl.java:canInitVertex(3195)) - Cannot init vertex: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] numTasks: 1 numUnitializedEdges: 0 numInitializedInputs: 0 initWaitsForRootInitializers: true
2016-01-14 17:40:20,221 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793219985_0001_1_00, eventCount=1
2016-01-14 17:40:20,222 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:20,223 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_INITIALIZED]: vertexName=F7D6C6EDE1324681B48381DE1AA02837, vertexId=vertex_1452793219985_0001_1_00, initRequestedTime=1452793220193, initedTime=1452793220222, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=1
2016-01-14 17:40:20,223 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] with distanceFromRoot: 0
2016-01-14 17:40:20,223 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] transitioned from INITIALIZING to INITED due to event V_ROOT_INPUT_INITIALIZED
2016-01-14 17:40:20,224 INFO  impl.VertexImpl (VertexImpl.java:assignVertexManager(2203)) - Setting vertexManager to ShuffleVertexManager for vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,227 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:initialize(696)) - Shuffle Vertex Manager: settings minFrac:0.25 maxFrac:0.75 auto:false desiredTaskIput:104857600 minTasks:1
2016-01-14 17:40:20,227 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3014)) - Creating 1 tasks for vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,228 INFO  impl.VertexImpl (VertexImpl.java:handleInitEvent(3026)) - Directly initializing vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,228 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1915)) - Invoking committer inits for vertex, vertexId=vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,228 INFO  impl.VertexImpl (VertexImpl.java:initializeCommitters(1927)) - Instantiating committer for output=9D6DA63324B644CFB67E02568AD25461, vertexId=vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD], committerClass=org.apache.tez.mapreduce.committer.MROutputCommitter
2016-01-14 17:40:20,229 INFO  impl.VertexImpl (VertexImpl.java:run(1945)) - Invoking committer init for output=9D6DA63324B644CFB67E02568AD25461, vertexId=vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,231 INFO  committer.MROutputCommitter (MROutputCommitter.java:getOutputCommitter(145)) - OutputCommitter for outputName=9D6DA63324B644CFB67E02568AD25461, vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, outputCommitterClass=org.apache.hadoop.mapred.FileOutputCommitter
2016-01-14 17:40:20,231 INFO  impl.VertexImpl (VertexImpl.java:run(1949)) - Invoking committer setup for output=9D6DA63324B644CFB67E02568AD25461, vertexId=vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,232 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_INITIALIZED to history file
2016-01-14 17:40:20,233 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_INITIALIZED]: vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, vertexId=vertex_1452793219985_0001_1_03, initRequestedTime=1452793220223, initedTime=1452793220232, numTasks=1, processorName=cascading.flow.tez.FlowProcessor, additionalInputsCount=0
2016-01-14 17:40:20,233 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] transitioned from NEW to INITED due to event V_INIT
2016-01-14 17:40:20,233 INFO  impl.ImmediateStartVertexManager (ImmediateStartVertexManager.java:scheduleTasks(99)) - Starting 1 in F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,234 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,234 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:20,235 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793219985_0001_1_00, startRequestedTime=1452793220233, startedTime=1452793220233
2016-01-14 17:40:20,235 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:20,235 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean] submitted tez dag to app master: application_1452793219985_0001, with dag id: dag_1452793219985_0001_1
2016-01-14 17:40:20,235 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:20,236 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_STARTED]: vertexName=F7D6C6EDE1324681B48381DE1AA02837, taskId=task_1452793219985_0001_1_00_000000, scheduledTime=1452793220235, launchTime=1452793220235
2016-01-14 17:40:20,236 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_00_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:20,240 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793219985_0001_1_00 for vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] numStartedSources: 1 numSources: 1
2016-01-14 17:40:20,240 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] with distanceFromRoot: 1
2016-01-14 17:40:20,241 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,241 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: F7D6C6EDE1324681B48381DE1AA02837 in vertex: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,241 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: B82115378D004563BEC7B1F7370537D3 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:20,242 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:20,242 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793219985_0001_1_01, startRequestedTime=1452793220241, startedTime=1452793220241
2016-01-14 17:40:20,242 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:20,243 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: F7D6C6EDE1324681B48381DE1AA02837, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=0DA26E84B8D2442A8DA9F5E16634017A, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=B82115378D004563BEC7B1F7370537D3, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:20,244 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_00_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:20,244 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793219985_0001_1_01 for vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] numStartedSources: 1 numSources: 1
2016-01-14 17:40:20,245 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] with distanceFromRoot: 2
2016-01-14 17:40:20,245 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:20,246 INFO  node.AMNodeTracker (AMNodeTracker.java:nodeSeen(91)) - Adding new node: 127.0.0.1:0
2016-01-14 17:40:20,247 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: B82115378D004563BEC7B1F7370537D3 in vertex: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,247 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: D9CD3ADDFEC14C0499A0B6A80DA79DA7 with 1 source tasks and 1 pending tasks
2016-01-14 17:40:20,247 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:20,247 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793219985_0001_1_02, startRequestedTime=1452793220245, startedTime=1452793220245
2016-01-14 17:40:20,248 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:20,248 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(112)) - Adding #1 tokens and #0 secret keys for NM use for launching container
2016-01-14 17:40:20,249 INFO  container.AMContainerHelpers (AMContainerHelpers.java:createCommonContainerLaunchContext(122)) - Putting shuffle token in serviceData
2016-01-14 17:40:20,250 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793219985_0001_00_000001
2016-01-14 17:40:20,251 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:20,252 INFO  impl.VertexImpl (VertexImpl.java:transition(3155)) - Source vertex started: vertex_1452793219985_0001_1_02 for vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] numStartedSources: 1 numSources: 1
2016-01-14 17:40:20,252 INFO  impl.VertexImpl (VertexImpl.java:startIfPossible(3061)) - Triggering start event for vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] with distanceFromRoot: 3
2016-01-14 17:40:20,253 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:handleVertexStateUpdate(720)) - Received configured notification : CONFIGURED for vertex: D9CD3ADDFEC14C0499A0B6A80DA79DA7 in vertex: FDB3C4710CD245169EDFBD5314D6B5DD
2016-01-14 17:40:20,254 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:onVertexStarted(355)) - OnVertexStarted vertex: FDB3C4710CD245169EDFBD5314D6B5DD with 1 source tasks and 1 pending tasks
2016-01-14 17:40:20,254 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_STARTED to history file
2016-01-14 17:40:20,254 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_STARTED]: vertexId=vertex_1452793219985_0001_1_03, startRequestedTime=1452793220253, startedTime=1452793220253
2016-01-14 17:40:20,254 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793219985_0001_00_000001
2016-01-14 17:40:20,255 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - DAGAppMaster metrics system already initialized!
2016-01-14 17:40:20,254 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] transitioned from INITED to RUNNING due to event V_START
2016-01-14 17:40:20,256 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:20,256 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:20,256 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793219985_0001_00_000001, launchTime=1452793220255
2016-01-14 17:40:20,257 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000001
2016-01-14 17:40:20,257 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793219985_0001_1_00_000000_0] to container: [container_1452793219985_0001_00_000001]
2016-01-14 17:40:20,257 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:20,258 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793219985_0001_00_000001 given task: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,258 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000001: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,258 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,259 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,259 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793219985_0001_1_00_000000_0] started. Is using containerId: [container_1452793219985_0001_00_000001] on NM: [127.0.0.1:0]
2016-01-14 17:40:20,259 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:20,259 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:20,259 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=F7D6C6EDE1324681B48381DE1AA02837, taskAttemptId=attempt_1452793219985_0001_1_00_000000_0, startTime=1452793220258, containerId=container_1452793219985_0001_00_000001, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793219985_0001_00_000001/teamcity, completedLogs=
2016-01-14 17:40:20,259 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:20,260 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_00_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:20,260 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_00_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:20,260 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:20,261 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:20,261 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:20,261 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: F7D6C6EDE1324681B48381DE1AA02837, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_00_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=0DA26E84B8D2442A8DA9F5E16634017A, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }}, ], outputSpecList=[{{ destinationVertexName=B82115378D004563BEC7B1F7370537D3, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:20,261 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:20,262 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,262 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:20,262 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,263 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=0DA26E84B8D2442A8DA9F5E16634017A, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInput }
2016-01-14 17:40:20,263 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=B82115378D004563BEC7B1F7370537D3, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:20,263 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,264 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:20,264 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,265 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:20,266 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,264 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793219985_0001_1_00_000000_0 sent events: (0-1)
2016-01-14 17:40:20,264 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:20,266 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,266 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: INPUT, componentVertexName: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,266 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,268 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:20,272 INFO  input.MRInput (MRInput.java:initialize(440)) - Using New mapreduce API: false, split information via event: true
2016-01-14 17:40:20,273 INFO  input.MRInput (MRInput.java:initializeInternal(493)) - Initialzed MRInput: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,273 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,278 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: F7D6C6EDE1324681B48381DE1AA02837, ordinal: 0
2016-01-14 17:40:20,278 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: F7D6C6EDE1324681B48381DE1AA02837, mem on start (mb), free: 247, total: 453, max: 672
2016-01-14 17:40:20,278 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,278 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:20,278 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:20,279 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:20,279 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:20,279 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:20,279 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.input.MRInput
2016-01-14 17:40:20,280 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:20,280 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:20,280 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 104857600, TotalRequestedScaled: 9.679163076923077E7, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.15
2016-01-14 17:40:20,280 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, B82115378D004563BEC7B1F7370537D3, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=104857600
2016-01-14 17:40:20,281 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, 0DA26E84B8D2442A8DA9F5E16634017A, org.apache.tez.mapreduce.input.MRInput: requested=0, allocated=0
2016-01-14 17:40:20,281 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:20,281 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,281 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:20,281 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,282 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: 0DA26E84B8D2442A8DA9F5E16634017A being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:20,282 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:20,282 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:20,282 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:20,283 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,283 INFO  input.MRInput (MRInput.java:initFromEventInternal(619)) - Initializing RecordReader from event
2016-01-14 17:40:20,283 INFO  input.MRInput (MRInput.java:initFromEventInternal(630)) - Split Details -> SplitClass: org.apache.hadoop.mapred.split.TezGroupedSplit, OldSplit: org.apache.hadoop.mapred.split.TezGroupedSplit@5104c8be
2016-01-14 17:40:20,287 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:20,287 INFO  lib.MRReaderMapred (MRReaderMapred.java:setIncrementalConfigParams(163)) - Processing split: org.apache.hadoop.mapred.split.TezGroupedSplit@5104c8be
2016-01-14 17:40:20,287 INFO  input.MRInput (MRInput.java:initFromEventInternal(634)) - Initialized RecordReader from event
2016-01-14 17:40:20,288 INFO  input.MRInput (MRInput.java:processSplitEvent(586)) - Notifying on RecordReader Initialized
2016-01-14 17:40:20,287 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"] streamed: true, id: 0DA26E84B8D2442A8DA9F5E16634017A
2016-01-14 17:40:20,288 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]], id: 78266E84E2844E9B86EC71E4EB8B10E0
2016-01-14 17:40:20,288 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]] 78266E84E2844E9B86EC71E4EB8B10E0
2016-01-14 17:40:20,289 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@114035d; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@48e8cf38; comparator=cascading.tuple.hadoop.util.TupleComparator@2484e46c; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:20,289 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:20,290 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:20,290 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:20,303 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 26214396(104857584)
2016-01-14 17:40:20,303 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 100
2016-01-14 17:40:20,303 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 83886080
2016-01-14 17:40:20,303 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 104857600
2016-01-14 17:40:20,303 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 26214396; length = 6553600
2016-01-14 17:40:20,304 INFO  element.TezSourceStage (TezSourceStage.java:prepare(63)) - calling MRInput#start() on: Hfs["TextLine[['offset', 'line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/classes/test/data/critics.txt"]
2016-01-14 17:40:20,304 INFO  input.MRInput (MRInput.java:start(450)) - MRInput setup to received events1
2016-01-14 17:40:20,305 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: F7D6C6EDE1324681B48381DE1AA02837, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:20,308 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,309 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: F7D6C6EDE1324681B48381DE1AA02837, mem on close (mb), free: 145, total: 453, max: 672
2016-01-14 17:40:20,309 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:20,309 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:20,309 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 1676; bufvoid = 104857600
2016-01-14 17:40:20,309 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 26214396(104857584); kvend = 26214260(104857040); length = 137/6553600
2016-01-14 17:40:20,311 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,313 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793219985_0001_1_00_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:20,314 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793219985_0001_1_00, eventCount=1
2016-01-14 17:40:20,314 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 27 [[File System Counters FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0][cascading.flow.SliceCounters Process_Begin_Time=1452793220305, Process_Duration=3, Process_End_Time=1452793220308, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=1][cascading.flow.StepCounters Tuples_Read=7]]
2016-01-14 17:40:20,315 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:20,315 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=F7D6C6EDE1324681B48381DE1AA02837, taskAttemptId=attempt_1452793219985_0001_1_00_000000_0, startTime=1452793220258, finishTime=1452793220314, timeTaken=56, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220305, Process_Duration=3, Process_End_Time=1452793220308, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=1, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:20,315 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:20,316 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,316 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_00_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:20,316 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000001
2016-01-14 17:40:20,316 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20,318 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_FINISHED]: vertexName=F7D6C6EDE1324681B48381DE1AA02837, taskId=task_1452793219985_0001_1_00_000000, startTime=1452793220258, finishTime=1452793220317, timeTaken=59, status=SUCCEEDED, successfulAttemptID=attempt_1452793219985_0001_1_00_000000_0, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220305, Process_Duration=3, Process_End_Time=1452793220308, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=1, cascading.flow.StepCounters, Tuples_Read=7
2016-01-14 17:40:20,318 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_00_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:20,316 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:20,318 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:20,318 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] : 1
2016-01-14 17:40:20,319 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,320 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:20,321 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_FINISHED]: vertexName=F7D6C6EDE1324681B48381DE1AA02837, vertexId=vertex_1452793219985_0001_1_00, initRequestedTime=1452793220193, initedTime=1452793220222, startRequestedTime=1452793220233, startedTime=1452793220233, finishTime=1452793220319, timeTaken=86, status=SUCCEEDED, diagnostics=, counters=Counters: 28, org.apache.tez.common.counters.DAGCounter, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=1274, FILE_BYTES_WRITTEN=1641, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, SPILLED_RECORDS=35, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=35, OUTPUT_BYTES=1676, OUTPUT_BYTES_WITH_OVERHEAD=1085, OUTPUT_BYTES_PHYSICAL=1089, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=0, ADDITIONAL_SPILL_COUNT=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220305, Process_Duration=3, Process_End_Time=1452793220308, Read_Duration=1, Tuples_Read=7, Tuples_Written=35, Write_Duration=1, cascading.flow.StepCounters, Tuples_Read=7, vertexStats=firstTaskStartTime=1452793220258, firstTasksToStart=[ task_1452793219985_0001_1_00_000000 ], lastTaskFinishTime=1452793220314, lastTasksToFinish=[ task_1452793219985_0001_1_00_000000 ], minTaskDuration=56, maxTaskDuration=56, avgTaskDuration=56.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793219985_0001_1_00_000000 ], longestDurationTasks=[ task_1452793219985_0001_1_00_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:20,321 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:20,321 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] attempt: attempt_1452793219985_0001_1_00_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:20,321 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,321 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,321 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:20,322 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793219985_0001_00_000001, stoppedTime=1452793220322, exitStatus=0
2016-01-14 17:40:20,322 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:20,322 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837] completed., numCompletedVertices=1, numSuccessfulVertices=1, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:20,322 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:20,322 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_STARTED]: vertexName=B82115378D004563BEC7B1F7370537D3, taskId=task_1452793219985_0001_1_01_000000, scheduledTime=1452793220322, launchTime=1452793220322
2016-01-14 17:40:20,323 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:20,323 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_01_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:20,323 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793219985_0001_00_000001. Relying on regular task shutdown for it to end
2016-01-14 17:40:20,323 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,323 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:20,323 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: B82115378D004563BEC7B1F7370537D3, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=F7D6C6EDE1324681B48381DE1AA02837, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:20,324 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_01_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:20,324 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:20,325 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793219985_0001_00_000002
2016-01-14 17:40:20,325 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:20,326 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793219985_0001_00_000002
2016-01-14 17:40:20,326 WARN  impl.MetricsSystemImpl (MetricsSystemImpl.java:init(151)) - DAGAppMaster metrics system already initialized!
2016-01-14 17:40:20,327 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:20,327 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793219985_0001_00_000001 is valid, but no longer registered, and will be killed
2016-01-14 17:40:20,327 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000001: 10 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:20,328 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:20,328 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793219985_0001_00_000001, Exiting
2016-01-14 17:40:20,328 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793219985_0001_00_000001
2016-01-14 17:40:20,328 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793219985_0001_00_000001
2016-01-14 17:40:20,329 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793219985_0001_00_000002, launchTime=1452793220327
2016-01-14 17:40:20,330 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000002
2016-01-14 17:40:20,330 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793219985_0001_00_000001 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:20,330 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793219985_0001_00_000001 completed successfully
2016-01-14 17:40:20,331 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793219985_0001_1_01_000000_0] to container: [container_1452793219985_0001_00_000002]
2016-01-14 17:40:20,331 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000001 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:20,331 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:20,331 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793219985_0001_00_000002 given task: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,331 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000002: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,331 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793219985_0001_1_01_000000_0] started. Is using containerId: [container_1452793219985_0001_00_000002] on NM: [127.0.0.1:0]
2016-01-14 17:40:20,332 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=B82115378D004563BEC7B1F7370537D3, taskAttemptId=attempt_1452793219985_0001_1_01_000000_0, startTime=1452793220331, containerId=container_1452793219985_0001_00_000002, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793219985_0001_00_000002/teamcity, completedLogs=
2016-01-14 17:40:20,332 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_01_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,332 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,332 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:20,332 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,332 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_01_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:20,332 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:20,333 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:20,333 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:20,333 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:20,333 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:20,333 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: B82115378D004563BEC7B1F7370537D3, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_01_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=F7D6C6EDE1324681B48381DE1AA02837, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:20,333 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:20,334 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,334 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:20,334 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,337 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793219985_0001_1_01_000000_0 sent events: (0-1)
2016-01-14 17:40:20,337 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,337 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=F7D6C6EDE1324681B48381DE1AA02837, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:20,337 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:20,338 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:20,338 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,338 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:20,339 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,341 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:20,342 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,342 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,343 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:20,344 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:20,344 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:20,345 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,345 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,351 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: B82115378D004563BEC7B1F7370537D3, ordinal: 1
2016-01-14 17:40:20,351 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: B82115378D004563BEC7B1F7370537D3, mem on start (mb), free: 140, total: 453, max: 672
2016-01-14 17:40:20,351 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,351 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:20,352 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:20,352 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:20,352 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:20,352 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:20,352 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:20,352 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:20,353 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:20,353 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, D9CD3ADDFEC14C0499A0B6A80DA79DA7, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:20,353 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, F7D6C6EDE1324681B48381DE1AA02837, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:20,353 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:20,353 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: F7D6C6EDE1324681B48381DE1AA02837 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:20,353 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,353 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:20,354 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:20,355 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:20,358 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:20,359 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: F7D6C6EDE1324681B48381DE1AA02837 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:20,359 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:20,359 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:20,359 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:20,360 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,360 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:20,361 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,361 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793219985_0001_1_00_000000_0_10039, runDuration: 47754, ]
2016-01-14 17:40:20,363 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793219985_0001_1_00_000000_0_10039] to DISK_DIRECT, CompressedSize=1089, DecompressedSize=1085,EndTime=1452793220363, TimeTaken=1, Rate=1.04 MB/s
2016-01-14 17:40:20,363 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,363 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:20,363 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [F7D6C6EDE1324681B48381DE1AA02837] #10 in 1ms
2016-01-14 17:40:20,363 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [F7D6C6EDE1324681B48381DE1AA02837] #10, status:false, isInterrupted:false
2016-01-14 17:40:20,364 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:20,365 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]]
2016-01-14 17:40:20,365 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 1089 bytes from disk
2016-01-14 17:40:20,365 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:20,365 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:20,365 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 1061 bytes
2016-01-14 17:40:20,365 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]] streamed: true, id: 78266E84E2844E9B86EC71E4EB8B10E0
2016-01-14 17:40:20,366 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,366 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(euclidean)[by:[{?}:ALL]], id: E86A441B3920418A9BE33BBE52343FB1
2016-01-14 17:40:20,366 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:20,366 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(euclidean)[by:[{?}:ALL]] E86A441B3920418A9BE33BBE52343FB1
2016-01-14 17:40:20,367 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@7021f4bc; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@454a9c27; comparator=cascading.tuple.hadoop.util.TupleComparator@2551e7fb; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:20,367 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:20,368 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:20,368 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:20,377 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:20,377 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:20,378 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:20,378 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:20,378 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:20,379 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: CoGroup(euclidean)[by:[{1}:'movie'][numSelfJoins:1]] 78266E84E2844E9B86EC71E4EB8B10E0, for 1 inputs
2016-01-14 17:40:20,380 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(89)) - attempting to load codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:20,380 INFO  collect.SpillableTupleList (SpillableTupleList.java:getCodecClass(94)) - found codec: org.apache.hadoop.io.compress.GzipCodec
2016-01-14 17:40:20,380 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: B82115378D004563BEC7B1F7370537D3, all 1 inputs ready in: 00:00:00.000
2016-01-14 17:40:20,381 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@5748cd59; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@6ca1bad3
2016-01-14 17:40:20,405 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,405 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: B82115378D004563BEC7B1F7370537D3, mem on close (mb), free: 73, total: 453, max: 672
2016-01-14 17:40:20,406 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: F7D6C6EDE1324681B48381DE1AA02837
2016-01-14 17:40:20,406 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:20,406 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:20,407 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:20,407 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:20,407 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:20,407 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:20,407 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:20,408 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 13104; bufvoid = 66060288
2016-01-14 17:40:20,408 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514376(66057504); length = 693/4128768
2016-01-14 17:40:20,414 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,419 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793219985_0001_1_01_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:20,420 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793219985_0001_1_01, eventCount=1
2016-01-14 17:40:20,420 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=5, MERGE_PHASE_TIME=8, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793220381, Process_Duration=24, Process_End_Time=1452793220405, Read_Duration=0, Tuples_Read=35, Tuples_Written=174, Write_Duration=5]]
2016-01-14 17:40:20,421 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=B82115378D004563BEC7B1F7370537D3, taskAttemptId=attempt_1452793219985_0001_1_01_000000_0, startTime=1452793220331, finishTime=1452793220420, timeTaken=89, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=5, MERGE_PHASE_TIME=8, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220381, Process_Duration=24, Process_End_Time=1452793220405, Read_Duration=0, Tuples_Read=35, Tuples_Written=174, Write_Duration=5
2016-01-14 17:40:20,421 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_01_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:20,421 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20,422 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_FINISHED]: vertexName=B82115378D004563BEC7B1F7370537D3, taskId=task_1452793219985_0001_1_01_000000, startTime=1452793220331, finishTime=1452793220422, timeTaken=91, status=SUCCEEDED, successfulAttemptID=attempt_1452793219985_0001_1_01_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=5, MERGE_PHASE_TIME=8, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220381, Process_Duration=24, Process_End_Time=1452793220405, Read_Duration=0, Tuples_Read=35, Tuples_Written=174, Write_Duration=5
2016-01-14 17:40:20,422 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_01_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:20,423 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] : 1
2016-01-14 17:40:20,423 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,421 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:20,422 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:20,421 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:20,423 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,423 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000002
2016-01-14 17:40:20,424 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:20,424 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_FINISHED]: vertexName=B82115378D004563BEC7B1F7370537D3, vertexId=vertex_1452793219985_0001_1_01, initRequestedTime=1452793220195, initedTime=1452793220201, startRequestedTime=1452793220241, startedTime=1452793220241, finishTime=1452793220423, timeTaken=182, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=1153, FILE_BYTES_WRITTEN=7551, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=5, REDUCE_INPUT_RECORDS=35, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=209, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=475529216, OUTPUT_RECORDS=174, OUTPUT_BYTES=13104, OUTPUT_BYTES_WITH_OVERHEAD=7515, OUTPUT_BYTES_PHYSICAL=7519, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=1089, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=1089, SHUFFLE_BYTES_DECOMPRESSED=1085, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=1089, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=5, MERGE_PHASE_TIME=8, FIRST_EVENT_RECEIVED=4, LAST_EVENT_RECEIVED=4, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220381, Process_Duration=24, Process_End_Time=1452793220405, Read_Duration=0, Tuples_Read=35, Tuples_Written=174, Write_Duration=5, vertexStats=firstTaskStartTime=1452793220331, firstTasksToStart=[ task_1452793219985_0001_1_01_000000 ], lastTaskFinishTime=1452793220420, lastTasksToFinish=[ task_1452793219985_0001_1_01_000000 ], minTaskDuration=89, maxTaskDuration=89, avgTaskDuration=89.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793219985_0001_1_01_000000 ], longestDurationTasks=[ task_1452793219985_0001_1_01_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:20,424 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:20,424 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] attempt: attempt_1452793219985_0001_1_01_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:20,425 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,425 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:20,425 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,425 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:20,425 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793219985_0001_00_000002, stoppedTime=1452793220425, exitStatus=0
2016-01-14 17:40:20,426 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:20,426 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3] completed., numCompletedVertices=2, numSuccessfulVertices=2, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:20,426 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:20,426 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_STARTED]: vertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, taskId=task_1452793219985_0001_1_02_000000, scheduledTime=1452793220426, launchTime=1452793220426
2016-01-14 17:40:20,426 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:20,426 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_02_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:20,426 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793219985_0001_00_000002. Relying on regular task shutdown for it to end
2016-01-14 17:40:20,427 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,427 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:20,427 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: D9CD3ADDFEC14C0499A0B6A80DA79DA7, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=B82115378D004563BEC7B1F7370537D3, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=FDB3C4710CD245169EDFBD5314D6B5DD, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:20,427 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_02_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:20,427 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:20,428 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793219985_0001_00_000003
2016-01-14 17:40:20,428 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:20,429 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793219985_0001_00_000003
2016-01-14 17:40:20,430 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:20,432 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:20,432 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:20,432 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793219985_0001_00_000003, launchTime=1452793220431
2016-01-14 17:40:20,432 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000003
2016-01-14 17:40:20,433 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793219985_0001_1_02_000000_0] to container: [container_1452793219985_0001_00_000003]
2016-01-14 17:40:20,433 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:20,433 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793219985_0001_00_000003 given task: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,433 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000003: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,434 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793219985_0001_1_02_000000_0] started. Is using containerId: [container_1452793219985_0001_00_000003] on NM: [127.0.0.1:0]
2016-01-14 17:40:20,434 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,434 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793219985_0001_00_000002 is valid, but no longer registered, and will be killed
2016-01-14 17:40:20,434 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000002: 10 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:20,434 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,435 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:20,435 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:20,435 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, taskAttemptId=attempt_1452793219985_0001_1_02_000000_0, startTime=1452793220434, containerId=container_1452793219985_0001_00_000003, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793219985_0001_00_000003/teamcity, completedLogs=
2016-01-14 17:40:20,435 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793219985_0001_00_000002, Exiting
2016-01-14 17:40:20,435 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793219985_0001_00_000002
2016-01-14 17:40:20,435 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793219985_0001_00_000002
2016-01-14 17:40:20,436 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_02_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:20,436 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_02_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:20,435 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:20,436 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793219985_0001_00_000002 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:20,437 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793219985_0001_00_000002 completed successfully
2016-01-14 17:40:20,437 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:20,437 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:20,437 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:20,438 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: D9CD3ADDFEC14C0499A0B6A80DA79DA7, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_02_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=B82115378D004563BEC7B1F7370537D3, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=FDB3C4710CD245169EDFBD5314D6B5DD, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }}, ]
2016-01-14 17:40:20,438 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:20,438 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000002 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:20,438 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,438 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:20,439 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,439 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,443 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=B82115378D004563BEC7B1F7370537D3, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:20,443 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:20,443 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,444 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793219985_0001_1_02_000000_0 sent events: (0-1)
2016-01-14 17:40:20,444 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=FDB3C4710CD245169EDFBD5314D6B5DD, physicalEdgeCount=1, outputClassName=org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput }
2016-01-14 17:40:20,445 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:20,445 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:20,445 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: FDB3C4710CD245169EDFBD5314D6B5DD
2016-01-14 17:40:20,446 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:20,446 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:20,447 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,447 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,448 INFO  impl.ExternalSorter (ExternalSorter.java:getInitialMemoryRequirement(293)) - Requested SortBufferSize (tez.runtime.io.sort.mb): 100
2016-01-14 17:40:20,448 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 104857600, type: OUTPUT, componentVertexName: FDB3C4710CD245169EDFBD5314D6B5DD
2016-01-14 17:40:20,449 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: FDB3C4710CD245169EDFBD5314D6B5DD
2016-01-14 17:40:20,454 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: D9CD3ADDFEC14C0499A0B6A80DA79DA7, ordinal: 2
2016-01-14 17:40:20,454 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: D9CD3ADDFEC14C0499A0B6A80DA79DA7, mem on start (mb), free: 67, total: 453, max: 672
2016-01-14 17:40:20,454 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,454 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:20,454 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:20,454 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:20,454 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:20,454 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:20,455 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_OUTPUT
2016-01-14 17:40:20,455 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:20,455 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 24, TotalRequested: 739036352, TotalRequestedScaled: 3.69518176E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:1.05
2016-01-14 17:40:20,455 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, B82115378D004563BEC7B1F7370537D3, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=405125774
2016-01-14 17:40:20,455 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, FDB3C4710CD245169EDFBD5314D6B5DD, org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput: requested=104857600, allocated=66985083
2016-01-14 17:40:20,455 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:20,455 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: B82115378D004563BEC7B1F7370537D3 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:20,456 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:20,456 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:20,456 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,457 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:20,458 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:20,458 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: B82115378D004563BEC7B1F7370537D3 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:20,458 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=405125774. Updated to: ShuffleMem=405125774, postMergeMem: 0
2016-01-14 17:40:20,458 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=405125774, maxSingleShuffleLimit=101281440, mergeThreshold=364613184, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:20,458 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:20,461 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,461 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:20,461 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,462 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793219985_0001_1_01_000000_0_10042, runDuration: 76271, ]
2016-01-14 17:40:20,464 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(euclidean)[by:[{?}:ALL]]
2016-01-14 17:40:20,465 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(euclidean)[by:[{?}:ALL]] streamed: true, id: E86A441B3920418A9BE33BBE52343FB1
2016-01-14 17:40:20,465 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: GroupBy(euclidean)[by:[{2}:'name1', 'name2']], id: 745FEFE143394058A319B382B04D42FA
2016-01-14 17:40:20,465 INFO  element.TezGroupGate (TezGroupGate.java:prepare(96)) - calling OrderedPartitionedKVOutput#start() on: GroupBy(euclidean)[by:[{2}:'name1', 'name2']] 745FEFE143394058A319B382B04D42FA
2016-01-14 17:40:20,468 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(167)) - keySerializer=cascading.tuple.hadoop.io.TupleSerializer@72e87f48; valueSerializer=cascading.tuple.hadoop.io.TupleSerializer@7dfc8ae7; comparator=cascading.tuple.hadoop.util.TupleComparator@4f03844f; conf=cascading.tuple.hadoop.TupleSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
2016-01-14 17:40:20,468 INFO  impl.ExternalSorter (ExternalSorter.java:<init>(224)) - Instantiating Partitioner: [cascading.tuple.tez.util.TuplePartitioner]
2016-01-14 17:40:20,468 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiatePartitioner(109)) - Using partitioner class: cascading.tuple.tez.util.TuplePartitioner
2016-01-14 17:40:20,468 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:20,477 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793219985_0001_1_01_000000_0_10042] to DISK_DIRECT, CompressedSize=7519, DecompressedSize=7515,EndTime=1452793220477, TimeTaken=1, Rate=7.17 MB/s
2016-01-14 17:40:20,478 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,478 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.01 MB/s)
2016-01-14 17:40:20,478 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [B82115378D004563BEC7B1F7370537D3] #11 in 2ms
2016-01-14 17:40:20,479 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [B82115378D004563BEC7B1F7370537D3] #11, status:false, isInterrupted:false
2016-01-14 17:40:20,484 INFO  dflt.DefaultSorter (DefaultSorter.java:setEquator(328)) - (EQUATOR) 0 kvi 16515068(66060272)
2016-01-14 17:40:20,484 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(150)) - tez.runtime.io.sort.mb: 63
2016-01-14 17:40:20,484 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(151)) - soft limit at 52848232
2016-01-14 17:40:20,484 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(152)) - bufstart = 0; bufvoid = 66060288
2016-01-14 17:40:20,484 INFO  dflt.DefaultSorter (DefaultSorter.java:<init>(153)) - kvstart = 16515068; length = 4128768
2016-01-14 17:40:20,484 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:20,485 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(euclidean)[by:[{?}:ALL]] E86A441B3920418A9BE33BBE52343FB1, for 1 inputs
2016-01-14 17:40:20,485 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 7519 bytes from disk
2016-01-14 17:40:20,485 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:20,485 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:20,486 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 7443 bytes
2016-01-14 17:40:20,486 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,486 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:20,487 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: D9CD3ADDFEC14C0499A0B6A80DA79DA7, all 1 inputs ready in: 00:00:00.002
2016-01-14 17:40:20,488 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@2c7bf31a; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@7d9546c6
2016-01-14 17:40:20,511 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,511 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: D9CD3ADDFEC14C0499A0B6A80DA79DA7, mem on close (mb), free: 194, total: 452, max: 672
2016-01-14 17:40:20,512 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: B82115378D004563BEC7B1F7370537D3
2016-01-14 17:40:20,512 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:20,512 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:20,512 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:20,512 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:20,512 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:20,513 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(589)) - Starting flush of map output
2016-01-14 17:40:20,513 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(607)) - Sorting & Spilling map output
2016-01-14 17:40:20,513 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(608)) - bufstart = 0; bufend = 7074; bufvoid = 66060288
2016-01-14 17:40:20,513 INFO  dflt.DefaultSorter (DefaultSorter.java:flush(610)) - kvstart = 16515068(66060272); kvend = 16514724(66058896); length = 345/4128768
2016-01-14 17:40:20,516 INFO  dflt.DefaultSorter (DefaultSorter.java:spill(829)) - Finished spill 0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,521 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793219985_0001_1_02_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:20,522 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_DATA_MOVEMENT_EVENTS_GENERATED]: vertexId=vertex_1452793219985_0001_1_02, eventCount=1
2016-01-14 17:40:20,522 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 49 [[File System Counters FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5679, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=6, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=21, MERGE_PHASE_TIME=28, FIRST_EVENT_RECEIVED=5, LAST_EVENT_RECEIVED=5][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793220487, Process_Duration=24, Process_End_Time=1452793220511, Read_Duration=6, Tuples_Read=174, Tuples_Written=87, Write_Duration=4]]
2016-01-14 17:40:20,523 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, taskAttemptId=attempt_1452793219985_0001_1_02_000000_0, startTime=1452793220434, finishTime=1452793220522, timeTaken=88, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5679, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=6, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=21, MERGE_PHASE_TIME=28, FIRST_EVENT_RECEIVED=5, LAST_EVENT_RECEIVED=5, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220487, Process_Duration=24, Process_End_Time=1452793220511, Read_Duration=6, Tuples_Read=174, Tuples_Written=87, Write_Duration=4
2016-01-14 17:40:20,523 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_02_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:20,523 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20,523 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:20,524 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,523 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:20,524 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_FINISHED]: vertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, taskId=task_1452793219985_0001_1_02_000000, startTime=1452793220434, finishTime=1452793220523, timeTaken=89, status=SUCCEEDED, successfulAttemptID=attempt_1452793219985_0001_1_02_000000_0, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5679, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=6, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=21, MERGE_PHASE_TIME=28, FIRST_EVENT_RECEIVED=5, LAST_EVENT_RECEIVED=5, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220487, Process_Duration=24, Process_End_Time=1452793220511, Read_Duration=6, Tuples_Read=174, Tuples_Written=87, Write_Duration=4
2016-01-14 17:40:20,524 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_02_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:20,524 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] : 1
2016-01-14 17:40:20,525 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:20,525 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000003
2016-01-14 17:40:20,524 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:20,525 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,527 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:20,527 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_FINISHED]: vertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, vertexId=vertex_1452793219985_0001_1_02, initRequestedTime=1452793220203, initedTime=1452793220219, startRequestedTime=1452793220245, startedTime=1452793220245, finishTime=1452793220526, timeTaken=281, status=SUCCEEDED, diagnostics=, counters=Counters: 49, File System Counters, FILE_BYTES_READ=7583, FILE_BYTES_WRITTEN=5679, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=86, REDUCE_INPUT_RECORDS=174, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=261, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=6, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=87, OUTPUT_BYTES=7074, OUTPUT_BYTES_WITH_OVERHEAD=5123, OUTPUT_BYTES_PHYSICAL=5127, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=7519, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=7519, SHUFFLE_BYTES_DECOMPRESSED=7515, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=7519, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=21, MERGE_PHASE_TIME=28, FIRST_EVENT_RECEIVED=5, LAST_EVENT_RECEIVED=5, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220487, Process_Duration=24, Process_End_Time=1452793220511, Read_Duration=6, Tuples_Read=174, Tuples_Written=87, Write_Duration=4, vertexStats=firstTaskStartTime=1452793220434, firstTasksToStart=[ task_1452793219985_0001_1_02_000000 ], lastTaskFinishTime=1452793220522, lastTasksToFinish=[ task_1452793219985_0001_1_02_000000 ], minTaskDuration=88, maxTaskDuration=88, avgTaskDuration=88.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793219985_0001_1_02_000000 ], longestDurationTasks=[ task_1452793219985_0001_1_02_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:20,527 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:20,528 INFO  impl.VertexImpl (VertexImpl.java:transition(3477)) - Source task attempt completed for vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] attempt: attempt_1452793219985_0001_1_02_000000_0 with state: SUCCEEDED vertexState: RUNNING
2016-01-14 17:40:20,528 INFO  vertexmanager.ShuffleVertexManager (ShuffleVertexManager.java:schedulePendingTasks(608)) - All source tasks assigned. Ramping up 1 remaining tasks for vertex: FDB3C4710CD245169EDFBD5314D6B5DD
2016-01-14 17:40:20,528 INFO  impl.VertexImpl (VertexImpl.java:scheduleTasks(1250)) - Routing pending task events for vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,528 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:20,528 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793219985_0001_00_000003, stoppedTime=1452793220528, exitStatus=0
2016-01-14 17:40:20,528 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:20,529 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:20,529 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7] completed., numCompletedVertices=3, numSuccessfulVertices=3, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:20,529 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_STARTED to history file
2016-01-14 17:40:20,530 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_STARTED]: vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, taskId=task_1452793219985_0001_1_03_000000, scheduledTime=1452793220529, launchTime=1452793220529
2016-01-14 17:40:20,530 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_03_000000 Task Transitioned from NEW to SCHEDULED due to event T_SCHEDULE
2016-01-14 17:40:20,531 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(244)) - Ignoring stop request for containerId container_1452793219985_0001_00_000003. Relying on regular task shutdown for it to end
2016-01-14 17:40:20,531 INFO  impl.DAGSchedulerNaturalOrderControlled (DAGSchedulerNaturalOrderControlled.java:scheduleTask(111)) - Scheduled vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,531 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from STOP_REQUESTED to STOPPING via event C_NM_STOP_SENT
2016-01-14 17:40:20,532 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1014)) - remoteTaskSpec:DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: FDB3C4710CD245169EDFBD5314D6B5DD, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=9D6DA63324B644CFB67E02568AD25461, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:20,532 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_03_000000_0 TaskAttempt Transitioned from NEW to START_WAIT due to event TA_SCHEDULE
2016-01-14 17:40:20,532 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_LAUNCH_REQUEST
2016-01-14 17:40:20,533 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(521)) - Sending Launch Request for Container with id: container_1452793219985_0001_00_000004
2016-01-14 17:40:20,533 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000004 transitioned from ALLOCATED to LAUNCHING via event C_LAUNCH_REQUEST
2016-01-14 17:40:20,535 INFO  task.TezChild (TezChild.java:newTezChild(432)) - PID, containerIdentifier:  null, container_1452793219985_0001_00_000004
2016-01-14 17:40:20,535 WARN  impl.MetricsConfig (MetricsConfig.java:loadFirst(124)) - Cannot locate configuration: tried hadoop-metrics2-teztask.properties,hadoop-metrics2.properties
2016-01-14 17:40:20,536 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793219985_0001_00_000003 is valid, but no longer registered, and will be killed
2016-01-14 17:40:20,536 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000003: 10 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:20,536 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793219985_0001_00_000003, Exiting
2016-01-14 17:40:20,537 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793219985_0001_00_000003
2016-01-14 17:40:20,537 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793219985_0001_00_000003
2016-01-14 17:40:20,537 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793219985_0001_00_000003 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:20,537 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793219985_0001_00_000003 completed successfully
2016-01-14 17:40:20,537 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_LAUNCHED to history file
2016-01-14 17:40:20,537 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_LAUNCHED]: containerId=container_1452793219985_0001_00_000004, launchTime=1452793220537
2016-01-14 17:40:20,538 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000004
2016-01-14 17:40:20,538 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000004 transitioned from LAUNCHING to IDLE via event C_LAUNCHED
2016-01-14 17:40:20,539 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000003 transitioned from STOPPING to COMPLETED via event C_COMPLETED
2016-01-14 17:40:20,539 INFO  container.AMContainerImpl (AMContainerImpl.java:transition(827)) - Assigned taskAttempt + [attempt_1452793219985_0001_1_03_000000_0] to container: [container_1452793219985_0001_00_000004]
2016-01-14 17:40:20,539 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000004 transitioned from IDLE to RUNNING via event C_PULL_TA
2016-01-14 17:40:20,539 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(240)) - Container with id: container_1452793219985_0001_00_000004 given task: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,540 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000004: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,540 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:transition(1146)) - TaskAttempt: [attempt_1452793219985_0001_1_03_000000_0] started. Is using containerId: [container_1452793219985_0001_00_000004] on NM: [127.0.0.1:0]
2016-01-14 17:40:20,540 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_STARTED]: vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, taskAttemptId=attempt_1452793219985_0001_1_03_000000_0, startTime=1452793220540, containerId=container_1452793219985_0001_00_000004, nodeId=127.0.0.1:0, inProgressLogs=127.0.0.1:0/node/containerlogs/container_1452793219985_0001_00_000004/teamcity, completedLogs=
2016-01-14 17:40:20,540 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_STARTED to history file
2016-01-14 17:40:20,540 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_03_000000_0 TaskAttempt Transitioned from START_WAIT to RUNNING due to event TA_STARTED_REMOTELY
2016-01-14 17:40:20,541 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_03_000000 Task Transitioned from SCHEDULED to RUNNING due to event T_ATTEMPT_LAUNCHED
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,541 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,541 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,541 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(273)) - Refreshing UGI since Credentials have changed
2016-01-14 17:40:20,541 INFO  task.TezChild (TezChild.java:handleNewTaskCredentials(276)) - Credentials : #Tokens=1, #SecretKeys=0
2016-01-14 17:40:20,542 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(302)) - Localizing additional local resources for Task : {}
2016-01-14 17:40:20,542 INFO  task.TezChild (TezChild.java:handleNewTaskLocalResources(312)) - Done localizing additional resources
2016-01-14 17:40:20,542 INFO  metrics.TaskCounterUpdater (TaskCounterUpdater.java:initResourceCalculatorPlugin(148)) -  Using ResourceCalculatorProcessTree : [ ]
2016-01-14 17:40:20,542 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:<init>(149)) - Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : [A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C] euclidean/(1/1) ...anceShort/euclidean/short, VertexName: FDB3C4710CD245169EDFBD5314D6B5DD, VertexParallelism: 1, TaskAttemptID:attempt_1452793219985_0001_1_03_000000_0, processorName=cascading.flow.tez.FlowProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }}, ], outputSpecList=[{{ destinationVertexName=9D6DA63324B644CFB67E02568AD25461, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ]
2016-01-14 17:40:20,542 INFO  resources.MemoryDistributor (MemoryDistributor.java:<init>(85)) - InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=704643072
2016-01-14 17:40:20,544 INFO  task.TezTaskRunner (TezTaskRunner.java:run(172)) - Initializing task, taskAttemptId=attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,544 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(191)) - Initializing LogicalProcessorIORuntimeTask
2016-01-14 17:40:20,545 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(195)) - Creating processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,548 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(389)) - Initializing Input using InputSpec: { sourceVertexName=D9CD3ADDFEC14C0499A0B6A80DA79DA7, physicalEdgeCount=1, inputClassName=org.apache.tez.runtime.library.input.OrderedGroupedKVInput }
2016-01-14 17:40:20,548 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(438)) - Initializing Output using OutputSpec: { destinationVertexName=9D6DA63324B644CFB67E02568AD25461, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }
2016-01-14 17:40:20,548 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(488)) - Initializing processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,549 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createOutput(565)) - Creating Output
2016-01-14 17:40:20,549 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(446)) - Initializing Output with dest edge: 9D6DA63324B644CFB67E02568AD25461
2016-01-14 17:40:20,550 INFO  output.MROutput (MROutput.java:initialize(347)) - Initializing Simple Output
2016-01-14 17:40:20,548 INFO  impl.TaskImpl (TaskImpl.java:getTaskAttemptTezEvents(481)) - TaskAttempt:attempt_1452793219985_0001_1_03_000000_0 sent events: (0-1)
2016-01-14 17:40:20,550 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 0, type: OUTPUT, componentVertexName: 9D6DA63324B644CFB67E02568AD25461
2016-01-14 17:40:20,549 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:createInput(537)) - Creating Input
2016-01-14 17:40:20,550 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(397)) - Initializing Input with src edge: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,551 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(88)) - cascading version: 3.0.4
2016-01-14 17:40:20,552 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(310)) - Initial Shuffle Memory Required: 634178752, based on INPUT_BUFFER_factor: 0.9
2016-01-14 17:40:20,552 INFO  orderedgrouped.MergeManager (MergeManager.java:getInitialMemoryRequirement(320)) - Initial Memory required for final merged output: 0, using factor: 0.0
2016-01-14 17:40:20,553 INFO  resources.MemoryDistributor$RequestorInfo (MemoryDistributor.java:<init>(233)) - Received request: 634178752, type: INPUT, componentVertexName: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,555 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(403)) - Initialized Input with src edge: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,562 INFO  output.MROutput (MROutput.java:initialize(418)) - Initialized Simple Output, using_new_api: false
2016-01-14 17:40:20,563 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(452)) - Initialized Output with dest edge: 9D6DA63324B644CFB67E02568AD25461
2016-01-14 17:40:20,564 INFO  tez.FlowProcessor (FlowProcessor.java:initialize(94)) - flow node id: FDB3C4710CD245169EDFBD5314D6B5DD, ordinal: 3
2016-01-14 17:40:20,564 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: FDB3C4710CD245169EDFBD5314D6B5DD, mem on start (mb), free: 186, total: 452, max: 672
2016-01-14 17:40:20,564 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initializeLogicalIOProcessor(491)) - Initialized processor, processorClassName=cascading.flow.tez.FlowProcessor
2016-01-14 17:40:20,564 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 2 initializers to finish
2016-01-14 17:40:20,564 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(220)) - Waiting for 1 initializers to finish
2016-01-14 17:40:20,564 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(233)) - All initializers finished
2016-01-14 17:40:20,564 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(124)) - Using Allocator class: org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor
2016-01-14 17:40:20,565 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:getRequestTypeForClass(197)) - Falling back to RequestType.OTHER for class: org.apache.tez.mapreduce.output.MROutput
2016-01-14 17:40:20,565 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 1, for type: OTHER
2016-01-14 17:40:20,565 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:initialProcessMemoryRequestContext(171)) - ScaleFactor: 12, for type: SORTED_MERGED_INPUT
2016-01-14 17:40:20,565 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:computeReservedFraction(262)) - InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996
2016-01-14 17:40:20,565 INFO  resources.WeightedScalingMemoryDistributor (WeightedScalingMemoryDistributor.java:assignMemory(126)) - Scaling Requests. NumRequests: 2, numScaledRequests: 13, TotalRequested: 634178752, TotalRequestedScaled: 5.853957710769231E8, TotalJVMHeap: 704643072, TotalAvailable: 472110858, TotalRequested/TotalJVMHeap:0.90
2016-01-14 17:40:20,565 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: OUTPUT, 9D6DA63324B644CFB67E02568AD25461, org.apache.tez.mapreduce.output.MROutput: requested=0, allocated=0
2016-01-14 17:40:20,567 INFO  resources.MemoryDistributor (MemoryDistributor.java:makeInitialAllocations(138)) - Informing: INPUT, D9CD3ADDFEC14C0499A0B6A80DA79DA7, org.apache.tez.runtime.library.input.OrderedGroupedKVInput: requested=634178752, allocated=472110858
2016-01-14 17:40:20,567 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(255)) - Starting Inputs/Outputs
2016-01-14 17:40:20,567 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(268)) - Input: D9CD3ADDFEC14C0499A0B6A80DA79DA7 being auto started by the framework. Subsequent instances will not be auto-started
2016-01-14 17:40:20,567 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(419)) - Starting Input with src edge: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,567 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(289)) - Num IOs determined for AutoStart: 1
2016-01-14 17:40:20,568 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(291)) - Waiting for 1 IOs to start
2016-01-14 17:40:20,568 INFO  common.TezRuntimeUtils (TezRuntimeUtils.java:instantiateCombiner(65)) - No combiner specified via tez.runtime.combiner.class. Combiner will not be used
2016-01-14 17:40:20,569 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(182)) - Shuffle assigned with 1 inputs, codec: NoneifileReadAhead: true
2016-01-14 17:40:20,570 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:<init>(158)) - ShuffleScheduler running for sourceVertex: D9CD3ADDFEC14C0499A0B6A80DA79DA7 with configuration: maxFetchFailuresBeforeReporting=5, reportReadErrorImmediately=true, maxFailedUniqueFetches=1, abortFailureLimit=30, maxMapRuntime=0, maxTaskOutputAtOnce=20
2016-01-14 17:40:20,570 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(221)) - InitialRequest: ShuffleMem=634178752, postMergeMem=0, RuntimeTotalAvailable=472110858. Updated to: ShuffleMem=472110858, postMergeMem: 0
2016-01-14 17:40:20,570 INFO  orderedgrouped.MergeManager (MergeManager.java:<init>(252)) - MergerManager: memoryLimit=472110858, maxSingleShuffleLimit=118027712, mergeThreshold=424899744, ioSortFactor=100, memToMemMergeOutputsThreshold=100
2016-01-14 17:40:20,571 INFO  orderedgrouped.Shuffle (Shuffle.java:<init>(233)) - Num fetchers being started: 1
2016-01-14 17:40:20,572 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:call(421)) - Started Input with src edge: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,573 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:initialize(304)) - AutoStartComplete
2016-01-14 17:40:20,573 INFO  task.TezTaskRunner (TezTaskRunner.java:run(175)) - Running task, taskAttemptId=attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,573 INFO  orderedgrouped.ShuffleInputEventHandlerOrderedGrouped (ShuffleInputEventHandlerOrderedGrouped.java:processDataMovementEvent(84)) - DME srcIdx: 0, targetIdx: 0, attemptNum: 0, payload: [host: ip-10-187-41-40, port: 0, pathComponent: attempt_1452793219985_0001_1_02_000000_0_10045, runDuration: 73796, ]
2016-01-14 17:40:20,576 INFO  graph.Hadoop2TezStreamGraph (Hadoop2TezStreamGraph.java:buildGraph(151)) - using streamed source: GroupBy(euclidean)[by:[{2}:'name1', 'name2']]
2016-01-14 17:40:20,577 INFO  tez.FlowProcessor (FlowProcessor.java:run(121)) - sourcing from: GroupBy(euclidean)[by:[{2}:'name1', 'name2']] streamed: true, id: 745FEFE143394058A319B382B04D42FA
2016-01-14 17:40:20,577 INFO  tez.FlowProcessor (FlowProcessor.java:run(124)) - sinking to: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceShort/euclidean/short"], id: 9D6DA63324B644CFB67E02568AD25461
2016-01-14 17:40:20,577 INFO  element.TezSinkStage (TezSinkStage.java:prepare(57)) - calling MROutput#start() on: Hfs["TextLine[['line']->[ALL]]"]["/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output/hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceShort/euclidean/short"]
2016-01-14 17:40:20,578 INFO  element.TezGroupGate (TezGroupGate.java:prepare(88)) - calling OrderedGroupedKVInput#start() on: GroupBy(euclidean)[by:[{2}:'name1', 'name2']] 745FEFE143394058A319B382B04D42FA, for 1 inputs
2016-01-14 17:40:20,578 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logIndividualFetchComplete(250)) - Completed fetch for attempt: InputAttemptIdentifier [inputIdentifier=InputIdentifier [inputIndex=0], attemptNumber=0, pathComponent=attempt_1452793219985_0001_1_02_000000_0_10045] to DISK_DIRECT, CompressedSize=5127, DecompressedSize=5123,EndTime=1452793220578, TimeTaken=3, Rate=1.63 MB/s
2016-01-14 17:40:20,579 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:copySucceeded(213)) - All inputs fetched for input vertex : D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,579 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:logProgress(263)) - copy(1 of 1. Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) 0.00 MB/s)
2016-01-14 17:40:20,579 INFO  orderedgrouped.ShuffleScheduler (ShuffleScheduler.java:freeHost(529)) - ip-10-187-41-40:0 freed by fetcher [D9CD3ADDFEC14C0499A0B6A80DA79DA7] #12 in 4ms
2016-01-14 17:40:20,579 INFO  orderedgrouped.Shuffle (Shuffle.java:cleanupFetchers(394)) - Shutdown..fetcher [D9CD3ADDFEC14C0499A0B6A80DA79DA7] #12, status:false, isInterrupted:false
2016-01-14 17:40:20,580 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 1 on-disk map-outputs
2016-01-14 17:40:20,580 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 1 files, 5127 bytes from disk
2016-01-14 17:40:20,580 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:20,581 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 1 sorted segments
2016-01-14 17:40:20,581 INFO  impl.TezMerger (TezMerger.java:merge(665)) - Down to the last merge-pass, with 1 segments left of total size: 5084 bytes
2016-01-14 17:40:20,581 INFO  orderedgrouped.Shuffle (Shuffle.java:call(381)) - merge complete for input vertex : D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,581 INFO  orderedgrouped.Shuffle (Shuffle.java:onSuccess(488)) - Shuffle Runner thread complete
2016-01-14 17:40:20,582 INFO  tez.FlowProcessor (FlowProcessor.java:waitForInputsReady(202)) - flow node id: FDB3C4710CD245169EDFBD5314D6B5DD, all 1 inputs ready in: 00:00:00.003
2016-01-14 17:40:20,582 INFO  common.ValuesIterator (ValuesIterator.java:<init>(83)) - keyDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@7e535d51; valueDeserializer=cascading.tuple.hadoop.io.TupleDeserializer@1313b1a2
2016-01-14 17:40:20,590 INFO  output.MROutput (MROutput.java:flush(534)) - Flushing Simple Output
2016-01-14 17:40:20,591 INFO  output.MROutput (MROutput.java:flush(544)) - Flushed Simple Output
2016-01-14 17:40:20,592 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:canCommit(353)) - Commit go/no-go request from attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,592 INFO  impl.TaskImpl (TaskImpl.java:canCommit(750)) - attempt_1452793219985_0001_1_03_000000_0 given a go for committing the task output.
2016-01-14 17:40:20,593 INFO  task.TezTaskRunner (TezTaskRunner.java:run(177)) - Closing task, taskAttemptId=attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,593 INFO  tez.FlowProcessor (LogUtil.java:logMemory(59)) - flow node id: FDB3C4710CD245169EDFBD5314D6B5DD, mem on close (mb), free: 184, total: 452, max: 672
2016-01-14 17:40:20,593 INFO  orderedgrouped.Shuffle (Shuffle.java:shutdown(320)) - Shutting down Shuffle for source: D9CD3ADDFEC14C0499A0B6A80DA79DA7
2016-01-14 17:40:20,594 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(854)) - finalMerge called with 0 in-memory map-outputs and 0 on-disk map-outputs
2016-01-14 17:40:20,594 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(951)) - Merging 0 files, 0 bytes from disk
2016-01-14 17:40:20,595 INFO  orderedgrouped.MergeManager (MergeManager.java:finalMerge(966)) - Merging 0 segments, 0 bytes from memory into reduce
2016-01-14 17:40:20,595 INFO  impl.TezMerger (TezMerger.java:merge(560)) - Merging 0 sorted segments
2016-01-14 17:40:20,595 INFO  impl.TezMerger (TezMerger.java:merge(562)) - Nothing to merge. Returning an empty iterator
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,596 INFO  task.TezTaskRunner (TezTaskRunner.java:run(181)) - Task completed, taskAttemptId=attempt_1452793219985_0001_1_03_000000_0, fatalErrorOccurred=false
2016-01-14 17:40:20,598 INFO  runtime.LogicalIOProcessorRuntimeTask (LogicalIOProcessorRuntimeTask.java:cleanup(712)) - Final Counters : Counters: 46 [[File System Counters FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0][org.apache.tez.common.counters.TaskCounter REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=12, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6][Shuffle Errors BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0][cascading.flow.SliceCounters Process_Begin_Time=1452793220582, Process_Duration=11, Process_End_Time=1452793220593, Read_Duration=2, Tuples_Read=87, Tuples_Written=21, Write_Duration=0][cascading.flow.StepCounters Tuples_Written=21]]
2016-01-14 17:40:20,598 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: 
2016-01-14 17:40:20,598 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,598 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_ATTEMPT_FINISHED to history file
2016-01-14 17:40:20,598 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, taskAttemptId=attempt_1452793219985_0001_1_03_000000_0, startTime=1452793220540, finishTime=1452793220598, timeTaken=58, status=SUCCEEDED, errorEnum=, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=12, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220582, Process_Duration=11, Process_End_Time=1452793220593, Read_Duration=2, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:20,599 INFO  impl.TaskAttemptImpl (TaskAttemptImpl.java:handle(692)) - attempt_1452793219985_0001_1_03_000000_0 TaskAttempt Transitioned from RUNNING to SUCCEEDED due to event TA_DONE
2016-01-14 17:40:20,599 INFO  impl.TaskImpl (TaskImpl.java:transition(1052)) - Task succeeded with attempt attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20,600 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event TASK_FINISHED to history file
2016-01-14 17:40:20,600 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:handleEvent(149)) - Processing the event EventType: S_TA_ENDED
2016-01-14 17:40:20,598 INFO  task.ContainerReporter (ContainerReporter.java:call(55)) - Attempting to fetch new task for container container_1452793219985_0001_00_000004
2016-01-14 17:40:20,600 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:TASK_FINISHED]: vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, taskId=task_1452793219985_0001_1_03_000000, startTime=1452793220540, finishTime=1452793220599, timeTaken=59, status=SUCCEEDED, successfulAttemptID=attempt_1452793219985_0001_1_03_000000_0, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=12, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220582, Process_Duration=11, Process_End_Time=1452793220593, Read_Duration=2, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21
2016-01-14 17:40:20,601 INFO  impl.TaskImpl (TaskImpl.java:handle(862)) - task_1452793219985_0001_1_03_000000 Task Transitioned from RUNNING to SUCCEEDED due to event T_ATTEMPT_SUCCEEDED
2016-01-14 17:40:20,601 INFO  impl.VertexImpl (VertexImpl.java:transition(3536)) - Num completed Tasks for vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] : 1
2016-01-14 17:40:20,601 INFO  impl.VertexImpl (VertexImpl.java:checkVertexForCompletion(1674)) - Vertex succeeded: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,602 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event VERTEX_FINISHED to history file
2016-01-14 17:40:20,602 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:VERTEX_FINISHED]: vertexName=FDB3C4710CD245169EDFBD5314D6B5DD, vertexId=vertex_1452793219985_0001_1_03, initRequestedTime=1452793220223, initedTime=1452793220232, startRequestedTime=1452793220253, startedTime=1452793220253, finishTime=1452793220601, timeTaken=348, status=SUCCEEDED, diagnostics=, counters=Counters: 46, File System Counters, FILE_BYTES_READ=5159, FILE_BYTES_WRITTEN=900, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=20, REDUCE_INPUT_RECORDS=87, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=87, NUM_SHUFFLED_INPUTS=1, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=1, GC_TIME_MILLIS=0, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=474480640, OUTPUT_RECORDS=21, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=5127, SHUFFLE_BYTES=5127, SHUFFLE_BYTES_DECOMPRESSED=5123, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=5127, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=10, MERGE_PHASE_TIME=12, FIRST_EVENT_RECEIVED=6, LAST_EVENT_RECEIVED=6, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=1452793220582, Process_Duration=11, Process_End_Time=1452793220593, Read_Duration=2, Tuples_Read=87, Tuples_Written=21, Write_Duration=0, cascading.flow.StepCounters, Tuples_Written=21, vertexStats=firstTaskStartTime=1452793220540, firstTasksToStart=[ task_1452793219985_0001_1_03_000000 ], lastTaskFinishTime=1452793220598, lastTasksToFinish=[ task_1452793219985_0001_1_03_000000 ], minTaskDuration=58, maxTaskDuration=58, avgTaskDuration=58.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1452793219985_0001_1_03_000000 ], longestDurationTasks=[ task_1452793219985_0001_1_03_000000 ], vertexTaskStats={numFailedTasks=0, numSucceededTasks=1, numKilledTaskAttempts=0, numKilledTasks=0, numFailedTaskAttempts=0, numCompletedTasks=1}
2016-01-14 17:40:20,603 INFO  impl.VertexImpl (VertexImpl.java:handle(1564)) - vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED
2016-01-14 17:40:20,603 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000004 transitioned from RUNNING to IDLE via event C_TA_SUCCEEDED
2016-01-14 17:40:20,603 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:CONTAINER_STOPPED]: containerId=container_1452793219985_0001_00_000004, stoppedTime=1452793220603, exitStatus=0
2016-01-14 17:40:20,604 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000004 transitioned from IDLE to STOP_REQUESTED via event C_STOP_REQUEST
2016-01-14 17:40:20,604 INFO  impl.DAGImpl (DAGImpl.java:transition(1747)) - Vertex vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD] completed., numCompletedVertices=4, numSuccessfulVertices=4, numFailedVertices=0, numKilledVertices=0, numVertices=4
2016-01-14 17:40:20,605 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(831)) - Calling DAG commit/abort for dag: dag_1452793219985_0001_1
2016-01-14 17:40:20,605 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:DAG_COMMIT_STARTED]: dagID=dag_1452793219985_0001_1
2016-01-14 17:40:20,605 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(892)) - Committing output: 9D6DA63324B644CFB67E02568AD25461 for vertex: vertex_1452793219985_0001_1_03 [FDB3C4710CD245169EDFBD5314D6B5DD]
2016-01-14 17:40:20,604 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event CONTAINER_STOPPED to history file
2016-01-14 17:40:20,611 INFO  app.TaskAttemptListenerImpTezDag (TaskAttemptListenerImpTezDag.java:getTask(213)) - Container with id: container_1452793219985_0001_00_000004 is valid, but no longer registered, and will be killed
2016-01-14 17:40:20,611 INFO  task.ContainerReporter (ContainerReporter.java:call(65)) - Got TaskUpdate for containerId= container_1452793219985_0001_00_000004: 11 ms after starting to poll. TaskInfo: shouldDie: true
2016-01-14 17:40:20,611 INFO  task.TezChild (TezChild.java:run(207)) - ContainerTask returned shouldDie=true for container container_1452793219985_0001_00_000004, Exiting
2016-01-14 17:40:20,611 INFO  task.TezChild (TezChild.java:shutdown(343)) - Shutdown invoked for container container_1452793219985_0001_00_000004
2016-01-14 17:40:20,611 INFO  task.TezChild (TezChild.java:shutdown(345)) - Shutting down container container_1452793219985_0001_00_000004
2016-01-14 17:40:20,611 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(270)) - ContainerExecutionResult for: container_1452793219985_0001_00_000004 = ContainerExecutionResult{exitStatus=SUCCESS, throwable=null, errorMessage='Asked to die by the AM'}
2016-01-14 17:40:20,611 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:onSuccess(274)) - Container: container_1452793219985_0001_00_000004 completed successfully
2016-01-14 17:40:20,616 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793219985_0001_1_02 [D9CD3ADDFEC14C0499A0B6A80DA79DA7]
2016-01-14 17:40:20,616 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793219985_0001_1_01 [B82115378D004563BEC7B1F7370537D3]
2016-01-14 17:40:20,616 INFO  impl.DAGImpl (DAGImpl.java:commitOrAbortOutputs(871)) - No output committers for vertex: vertex_1452793219985_0001_1_00 [F7D6C6EDE1324681B48381DE1AA02837]
2016-01-14 17:40:20,619 INFO  recovery.RecoveryService (RecoveryService.java:handle(306)) - DAG completed, dagId=dag_1452793219985_0001_1, queueSize=0
2016-01-14 17:40:20,619 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:handleEvent(151)) - Writing event DAG_FINISHED to history file
2016-01-14 17:40:20,620 INFO  history.HistoryEventHandler (HistoryEventHandler.java:handleCriticalEvent(110)) - [HISTORY][DAG:dag_1452793219985_0001_1][Event:DAG_FINISHED]: dagId=dag_1452793219985_0001_1, startTime=1452793220193, finishTime=1452793220616, timeTaken=423, status=SUCCEEDED, diagnostics=, counters=Counters: 55, org.apache.tez.common.counters.DAGCounter, NUM_SUCCEEDED_TASKS=4, TOTAL_LAUNCHED_TASKS=4, RACK_LOCAL_TASKS=1, File System Counters, FILE_BYTES_READ=15169, FILE_BYTES_WRITTEN=15771, FILE_READ_OPS=0, FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, org.apache.tez.common.counters.TaskCounter, REDUCE_INPUT_GROUPS=111, REDUCE_INPUT_RECORDS=296, COMBINE_INPUT_RECORDS=0, SPILLED_RECORDS=592, NUM_SHUFFLED_INPUTS=3, NUM_SKIPPED_INPUTS=0, NUM_FAILED_SHUFFLE_INPUTS=0, MERGED_MAP_OUTPUTS=3, GC_TIME_MILLIS=6, CPU_MILLISECONDS=0, PHYSICAL_MEMORY_BYTES=0, VIRTUAL_MEMORY_BYTES=0, COMMITTED_HEAP_BYTES=1900019712, INPUT_RECORDS_PROCESSED=0, OUTPUT_RECORDS=317, OUTPUT_BYTES=21854, OUTPUT_BYTES_WITH_OVERHEAD=13723, OUTPUT_BYTES_PHYSICAL=13735, ADDITIONAL_SPILLS_BYTES_WRITTEN=0, ADDITIONAL_SPILLS_BYTES_READ=13735, ADDITIONAL_SPILL_COUNT=0, SHUFFLE_BYTES=13735, SHUFFLE_BYTES_DECOMPRESSED=13723, SHUFFLE_BYTES_TO_MEM=0, SHUFFLE_BYTES_TO_DISK=0, SHUFFLE_BYTES_DISK_DIRECT=13735, NUM_MEM_TO_DISK_MERGES=0, NUM_DISK_TO_DISK_MERGES=0, SHUFFLE_PHASE_TIME=36, MERGE_PHASE_TIME=48, FIRST_EVENT_RECEIVED=15, LAST_EVENT_RECEIVED=15, Shuffle Errors, BAD_ID=0, CONNECTION=0, IO_ERROR=0, WRONG_LENGTH=0, WRONG_MAP=0, WRONG_REDUCE=0, cascading.flow.SliceCounters, Process_Begin_Time=5811172881755, Process_Duration=62, Process_End_Time=5811172881817, Read_Duration=9, Tuples_Read=303, Tuples_Written=317, Write_Duration=10, cascading.flow.StepCounters, Tuples_Read=7, Tuples_Written=21
2016-01-14 17:40:20,620 INFO  impl.DAGImpl (DAGImpl.java:finished(1180)) - DAG: dag_1452793219985_0001_1 finished with state: SUCCEEDED
2016-01-14 17:40:20,620 INFO  impl.DAGImpl (DAGImpl.java:handle(973)) - dag_1452793219985_0001_1 transitioned from RUNNING to SUCCEEDED due to event DAG_VERTEX_COMPLETED
2016-01-14 17:40:20,620 INFO  launcher.LocalContainerLauncher (LocalContainerLauncher.java:stop(242)) - Ignoring stop request for containerId: container_1452793219985_0001_00_000004
2016-01-14 17:40:20,621 INFO  container.AMContainerImpl (AMContainerImpl.java:handle(442)) - AMContainer container_1452793219985_0001_00_000004 transitioned from STOP_REQUESTED to COMPLETED via event C_COMPLETED
2016-01-14 17:40:20 Completed Dag: dag_1452793219985_0001_1
2016-01-14 17:40:20,621 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(579)) - DAG completed, dagId=dag_1452793219985_0001_1, dagState=SUCCEEDED
2016-01-14 17:40:20,622 INFO  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(164)) - Redirecting log file based on addend: dag_1452793219985_0001_1_post
2016-01-14 17:40:20,622 WARN  common.TezUtilsInternal (TezUtilsInternal.java:updateLoggers(179)) - Not configured with appender named: CLA. Cannot reconfigure logger output
2016-01-14 17:40:20,622 INFO  app.DAGAppMaster (DAGAppMaster.java:handle(625)) - Waiting for next DAG to be submitted.
2016-01-14 17:40:20,630 INFO  client.TezClient (TezClient.java:stop(495)) - Shutting down Tez Session, sessionName=(1/1) ...anceShort/euclidean/short, applicationId=application_1452793219985_0001
2016-01-14 17:40:20,632 INFO  client.DAGClientHandler (DAGClientHandler.java:shutdownAM(117)) - Received message to shutdown AM
2016-01-14 17:40:20,632 INFO  rm.TaskSchedulerEventHandler (TaskSchedulerEventHandler.java:setShouldUnregisterFlag(580)) - TaskScheduler notified that it should unregister from RM
2016-01-14 17:40:20,633 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdownTezAM(1088)) - No current running DAG, shutting down the AM
2016-01-14 17:40:20,633 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(692)) - DAGAppMasterShutdownHandler invoked
2016-01-14 17:40:20,633 INFO  app.DAGAppMaster (DAGAppMaster.java:shutdown(701)) - Handling DAGAppMaster shutdown
2016-01-14 17:40:20,634 INFO  app.DAGAppMaster (DAGAppMaster.java:run(721)) - Sleeping for 5 seconds before shutting down
2016-01-14 17:40:20,634 INFO  history.HistoryEventHandler (HistoryEventHandler.java:serviceStop(79)) - Stopping HistoryEventHandler
2016-01-14 17:40:20,634 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(189)) - Stopping RecoveryService
2016-01-14 17:40:20,635 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(192)) - Handle the remaining events in queue, queue size=0
2016-01-14 17:40:20,635 INFO  recovery.RecoveryService (RecoveryService.java:run(166)) - EventQueue take interrupted. Returning
2016-01-14 17:40:20,635 INFO  recovery.RecoveryService (RecoveryService.java:serviceStop(214)) - Closing Summary Stream
2016-01-14 17:40:20,636 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:serviceStop(118)) - Stopping SimpleHistoryLoggingService, eventQueueBacklog=0
2016-01-14 17:40:20,636 INFO  impl.SimpleHistoryLoggingService (SimpleHistoryLoggingService.java:run(105)) - EventQueue take interrupted. Returning
2016-01-14 17:40:20,642 INFO  app.DAGAppMaster (DAGAppMaster.java:run(1773)) - Completed deletion of tez scratch data dir, path=file:/opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/tmp/.staging/A5732A4646EF4DB18E8D54A24E36CF46/D2688C2332B44FE49C5A4383F2FFC80C/.tez/application_1452793219985_0001
2016-01-14 17:40:20,643 INFO  flow.Flow (BaseFlow.java:logInfo(1433)) - [euclidean]  completed in: 00:00.677
2016-01-14 17:40:20,722 INFO  cascading.PlatformTestCase (PlatformTestCase.java:tearDown(189)) - copying to local /opt/buildAgent-8.0.3/work/ca57560bc31094a7/apache-2.6.x-tez-0.6.x/build/test/output//hadoop2-tez/distanceusecaseplatform/testEuclideanDistanceShort/euclidean/short

2016-01-14 17:40:15 Running Dag: dag_1452793214302_0001_1
2016-01-14 17:40:15 Starting to run new task attempt: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_00_000000_0
2016-01-14 17:40:16 Starting to run new task attempt: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_01_000000_0
2016-01-14 17:40:16 Starting to run new task attempt: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_02_000000_0
2016-01-14 17:40:16 Starting to run new task attempt: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16 Completed running task attempt: attempt_1452793214302_0001_1_03_000000_0
2016-01-14 17:40:16 Completed Dag: dag_1452793214302_0001_1
2016-01-14 17:40:17 Running Dag: dag_1452793217481_0001_1
2016-01-14 17:40:17 Starting to run new task attempt: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17 Completed running task attempt: attempt_1452793217481_0001_1_00_000000_0
2016-01-14 17:40:17 Starting to run new task attempt: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:18 Completed running task attempt: attempt_1452793217481_0001_1_01_000000_0
2016-01-14 17:40:18 Starting to run new task attempt: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18 Completed running task attempt: attempt_1452793217481_0001_1_02_000000_0
2016-01-14 17:40:18 Starting to run new task attempt: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18 Completed running task attempt: attempt_1452793217481_0001_1_03_000000_0
2016-01-14 17:40:18 Completed Dag: dag_1452793217481_0001_1
2016-01-14 17:40:19 Running Dag: dag_1452793218807_0001_1
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_00_000000_0
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_01_000000_0
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_02_000000_0
2016-01-14 17:40:19 Starting to run new task attempt: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19 Completed running task attempt: attempt_1452793218807_0001_1_03_000000_0
2016-01-14 17:40:19 Completed Dag: dag_1452793218807_0001_1
2016-01-14 17:40:20 Running Dag: dag_1452793219985_0001_1
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_00_000000_0
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_01_000000_0
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_02_000000_0
2016-01-14 17:40:20 Starting to run new task attempt: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20 Completed running task attempt: attempt_1452793219985_0001_1_03_000000_0
2016-01-14 17:40:20 Completed Dag: dag_1452793219985_0001_1