PageRenderTime 26ms CodeModel.GetById 8ms app.highlight 13ms RepoModel.GetById 1ms app.codeStats 0ms

/boto-2.5.2/tests/unit/emr/test_emr_responses.py

#
Python | 373 lines | 345 code | 6 blank | 22 comment | 0 complexity | be83a998fbfcdefa6fc2fd55dccd4172 MD5 | raw file
  1# Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com>
  2#
  3# Permission is hereby granted, free of charge, to any person obtaining a
  4# copy of this software and associated documentation files (the
  5# "Software"), to deal in the Software without restriction, including
  6# without limitation the rights to use, copy, modify, merge, publish, dis-
  7# tribute, sublicense, and/or sell copies of the Software, and to permit
  8# persons to whom the Software is furnished to do so, subject to the fol-
  9# lowing conditions:
 10#
 11# The above copyright notice and this permission notice shall be included
 12# in all copies or substantial portions of the Software.
 13#
 14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
 16# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
 17# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 18# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 19# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 20# IN THE SOFTWARE.
 21
 22
 23# NOTE: These tests only cover the very simple cases I needed to test
 24# for the InstanceGroup fix.
 25
 26import xml.sax
 27import unittest
 28
 29from boto import handler
 30from boto.emr import emrobject
 31from boto.resultset import ResultSet
 32
 33
 34JOB_FLOW_EXAMPLE = """
 35<DescribeJobFlowsResponse
 36    xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15">
 37  <DescribeJobFlowsResult>
 38    <JobFlows>
 39      <member>
 40        <ExecutionStatusDetail>
 41          <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>
 42          <StartDateTime>2009-01-28T21:49:16Z</StartDateTime>
 43          <State>STARTING</State>
 44        </ExecutionStatusDetail>
 45        <Name>MyJobFlowName</Name>
 46        <LogUri>mybucket/subdir/</LogUri>
 47        <Steps>
 48          <member>
 49            <ExecutionStatusDetail>
 50              <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>
 51              <State>PENDING</State>
 52            </ExecutionStatusDetail>
 53            <StepConfig>
 54              <HadoopJarStep>
 55                <Jar>MyJarFile</Jar>
 56                <MainClass>MyMailClass</MainClass>
 57                <Args>
 58                  <member>arg1</member>
 59                  <member>arg2</member>
 60                </Args>
 61                <Properties/>
 62              </HadoopJarStep>
 63              <Name>MyStepName</Name>
 64              <ActionOnFailure>CONTINUE</ActionOnFailure>
 65            </StepConfig>
 66          </member>
 67        </Steps>
 68        <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId>
 69        <Instances>
 70          <Placement>
 71            <AvailabilityZone>us-east-1a</AvailabilityZone>
 72          </Placement>
 73          <SlaveInstanceType>m1.small</SlaveInstanceType>
 74          <MasterInstanceType>m1.small</MasterInstanceType>
 75          <Ec2KeyName>myec2keyname</Ec2KeyName>
 76          <InstanceCount>4</InstanceCount>
 77          <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps>
 78        </Instances>
 79      </member>
 80    </JobFlows>
 81  </DescribeJobFlowsResult>
 82  <ResponseMetadata>
 83    <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId>
 84  </ResponseMetadata>
 85</DescribeJobFlowsResponse>
 86"""
 87
 88JOB_FLOW_COMPLETED = """
 89<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
 90  <DescribeJobFlowsResult>
 91    <JobFlows>
 92      <member>
 93        <ExecutionStatusDetail>
 94          <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
 95          <LastStateChangeReason>Steps completed</LastStateChangeReason>
 96          <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>
 97          <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>
 98          <State>COMPLETED</State>
 99          <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>
100        </ExecutionStatusDetail>
101        <BootstrapActions/>
102        <Name>RealJobFlowName</Name>
103        <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri>
104        <Steps>
105          <member>
106            <StepConfig>
107              <HadoopJarStep>
108                <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar>
109                <Args>
110                  <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch</member>
111                </Args>
112                <Properties/>
113              </HadoopJarStep>
114              <Name>Setup Hadoop Debugging</Name>
115              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
116            </StepConfig>
117            <ExecutionStatusDetail>
118              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
119              <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>
120              <State>COMPLETED</State>
121              <EndDateTime>2010-10-21T01:04:22Z</EndDateTime>
122            </ExecutionStatusDetail>
123          </member>
124          <member>
125            <StepConfig>
126              <HadoopJarStep>
127                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
128                <Args>
129                  <member>-mapper</member>
130                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py</member>
131                  <member>-reducer</member>
132                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py</member>
133                  <member>-input</member>
134                  <member>s3://example.emrtest.data/raw/2010/10/20/*</member>
135                  <member>-input</member>
136                  <member>s3://example.emrtest.data/raw/2010/10/19/*</member>
137                  <member>-input</member>
138                  <member>s3://example.emrtest.data/raw/2010/10/18/*</member>
139                  <member>-input</member>
140                  <member>s3://example.emrtest.data/raw/2010/10/17/*</member>
141                  <member>-input</member>
142                  <member>s3://example.emrtest.data/raw/2010/10/16/*</member>
143                  <member>-input</member>
144                  <member>s3://example.emrtest.data/raw/2010/10/15/*</member>
145                  <member>-input</member>
146                  <member>s3://example.emrtest.data/raw/2010/10/14/*</member>
147                  <member>-output</member>
148                  <member>s3://example.emrtest.crunched/</member>
149                </Args>
150                <Properties/>
151              </HadoopJarStep>
152              <Name>testjob_Initial</Name>
153              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
154            </StepConfig>
155            <ExecutionStatusDetail>
156              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
157              <StartDateTime>2010-10-21T01:04:22Z</StartDateTime>
158              <State>COMPLETED</State>
159              <EndDateTime>2010-10-21T01:36:18Z</EndDateTime>
160            </ExecutionStatusDetail>
161          </member>
162          <member>
163            <StepConfig>
164              <HadoopJarStep>
165                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
166                <Args>
167                  <member>-mapper</member>
168                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py</member>
169                  <member>-reducer</member>
170                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py</member>
171                  <member>-input</member>
172                  <member>s3://example.emrtest.crunched/*</member>
173                  <member>-output</member>
174                  <member>s3://example.emrtest.step1/</member>
175                </Args>
176                <Properties/>
177              </HadoopJarStep>
178              <Name>testjob_step1</Name>
179              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
180            </StepConfig>
181            <ExecutionStatusDetail>
182              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
183              <StartDateTime>2010-10-21T01:36:18Z</StartDateTime>
184              <State>COMPLETED</State>
185              <EndDateTime>2010-10-21T01:37:51Z</EndDateTime>
186            </ExecutionStatusDetail>
187          </member>
188          <member>
189            <StepConfig>
190              <HadoopJarStep>
191                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
192                <Args>
193                  <member>-mapper</member>
194                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py</member>
195                  <member>-reducer</member>
196                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py</member>
197                  <member>-input</member>
198                  <member>s3://example.emrtest.crunched/*</member>
199                  <member>-output</member>
200                  <member>s3://example.emrtest.step2/</member>
201                </Args>
202                <Properties/>
203              </HadoopJarStep>
204              <Name>testjob_step2</Name>
205              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
206            </StepConfig>
207            <ExecutionStatusDetail>
208              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
209              <StartDateTime>2010-10-21T01:37:51Z</StartDateTime>
210              <State>COMPLETED</State>
211              <EndDateTime>2010-10-21T01:39:32Z</EndDateTime>
212            </ExecutionStatusDetail>
213          </member>
214          <member>
215            <StepConfig>
216              <HadoopJarStep>
217                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
218                <Args>
219                  <member>-mapper</member>
220                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py</member>
221                  <member>-reducer</member>
222                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py</member>
223                  <member>-input</member>
224                  <member>s3://example.emrtest.step1/*</member>
225                  <member>-output</member>
226                  <member>s3://example.emrtest.step3/</member>
227                </Args>
228                <Properties/>
229              </HadoopJarStep>
230              <Name>testjob_step3</Name>
231              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
232            </StepConfig>
233            <ExecutionStatusDetail>
234              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
235              <StartDateTime>2010-10-21T01:39:32Z</StartDateTime>
236              <State>COMPLETED</State>
237              <EndDateTime>2010-10-21T01:41:22Z</EndDateTime>
238            </ExecutionStatusDetail>
239          </member>
240          <member>
241            <StepConfig>
242              <HadoopJarStep>
243                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
244                <Args>
245                  <member>-mapper</member>
246                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py</member>
247                  <member>-reducer</member>
248                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py</member>
249                  <member>-input</member>
250                  <member>s3://example.emrtest.step1/*</member>
251                  <member>-output</member>
252                  <member>s3://example.emrtest.step4/</member>
253                </Args>
254                <Properties/>
255              </HadoopJarStep>
256              <Name>testjob_step4</Name>
257              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
258            </StepConfig>
259            <ExecutionStatusDetail>
260              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
261              <StartDateTime>2010-10-21T01:41:22Z</StartDateTime>
262              <State>COMPLETED</State>
263              <EndDateTime>2010-10-21T01:43:03Z</EndDateTime>
264            </ExecutionStatusDetail>
265          </member>
266        </Steps>
267        <JobFlowId>j-3H3Q13JPFLU22</JobFlowId>
268        <Instances>
269          <SlaveInstanceType>m1.large</SlaveInstanceType>
270          <MasterInstanceId>i-64c21609</MasterInstanceId>
271          <Placement>
272            <AvailabilityZone>us-east-1b</AvailabilityZone>
273          </Placement>
274          <InstanceGroups>
275            <member>
276              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
277              <InstanceRunningCount>0</InstanceRunningCount>
278              <StartDateTime>2010-10-21T01:02:09Z</StartDateTime>
279              <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime>
280              <State>ENDED</State>
281              <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>
282              <InstanceRequestCount>1</InstanceRequestCount>
283              <InstanceType>m1.large</InstanceType>
284              <Market>ON_DEMAND</Market>
285              <LastStateChangeReason>Job flow terminated</LastStateChangeReason>
286              <InstanceRole>MASTER</InstanceRole>
287              <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId>
288              <Name>master</Name>
289            </member>
290            <member>
291              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
292              <InstanceRunningCount>0</InstanceRunningCount>
293              <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>
294              <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>
295              <State>ENDED</State>
296              <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>
297              <InstanceRequestCount>9</InstanceRequestCount>
298              <InstanceType>m1.large</InstanceType>
299              <Market>ON_DEMAND</Market>
300              <LastStateChangeReason>Job flow terminated</LastStateChangeReason>
301              <InstanceRole>CORE</InstanceRole>
302              <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId>
303              <Name>slave</Name>
304            </member>
305          </InstanceGroups>
306          <NormalizedInstanceHours>40</NormalizedInstanceHours>
307          <HadoopVersion>0.20</HadoopVersion>
308          <MasterInstanceType>m1.large</MasterInstanceType>
309          <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</MasterPublicDnsName>
310          <Ec2KeyName>myubersecurekey</Ec2KeyName>
311          <InstanceCount>10</InstanceCount>
312          <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps>
313        </Instances>
314      </member>
315    </JobFlows>
316  </DescribeJobFlowsResult>
317  <ResponseMetadata>
318    <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId>
319  </ResponseMetadata>
320</DescribeJobFlowsResponse>
321"""
322
323
324class TestEMRResponses(unittest.TestCase):
325    def _parse_xml(self, body, markers):
326        rs = ResultSet(markers)
327        h = handler.XmlHandler(rs, None)
328        xml.sax.parseString(body, h)
329        return rs
330
331    def _assert_fields(self, response, **fields):
332        for field, expected in fields.items():
333            actual = getattr(response, field)
334            self.assertEquals(expected, actual,
335                              "Field %s: %r != %r" % (field, expected, actual))
336
337    def test_JobFlows_example(self):
338        [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE,
339                                    [('member', emrobject.JobFlow)])
340        self._assert_fields(jobflow,
341                            creationdatetime='2009-01-28T21:49:16Z',
342                            startdatetime='2009-01-28T21:49:16Z',
343                            state='STARTING',
344                            instancecount='4',
345                            jobflowid='j-3UN6WX5RRO2AG',
346                            loguri='mybucket/subdir/',
347                            name='MyJobFlowName',
348                            availabilityzone='us-east-1a',
349                            slaveinstancetype='m1.small',
350                            masterinstancetype='m1.small',
351                            ec2keyname='myec2keyname',
352                            keepjobflowalivewhennosteps='true')
353
354    def test_JobFlows_completed(self):
355        [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED,
356                                    [('member', emrobject.JobFlow)])
357        self._assert_fields(jobflow,
358                            creationdatetime='2010-10-21T01:00:25Z',
359                            startdatetime='2010-10-21T01:03:59Z',
360                            enddatetime='2010-10-21T01:44:18Z',
361                            state='COMPLETED',
362                            instancecount='10',
363                            jobflowid='j-3H3Q13JPFLU22',
364                            loguri='s3n://example.emrtest.scripts/jobflow_logs/',
365                            name='RealJobFlowName',
366                            availabilityzone='us-east-1b',
367                            slaveinstancetype='m1.large',
368                            masterinstancetype='m1.large',
369                            ec2keyname='myubersecurekey',
370                            keepjobflowalivewhennosteps='false')
371        self.assertEquals(6, len(jobflow.steps))
372        self.assertEquals(2, len(jobflow.instancegroups))
373