001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.coord;
020
021import java.util.Calendar;
022import java.util.Date;
023import java.util.HashMap;
024import java.util.Iterator;
025import java.util.List;
026import java.util.Map;
027
028import org.apache.hadoop.conf.Configuration;
029import org.apache.oozie.CoordinatorActionBean;
030import org.apache.oozie.command.coord.CoordCommandUtils;
031import org.apache.oozie.service.ELService;
032import org.apache.oozie.service.Services;
033import org.apache.oozie.util.DateUtils;
034import org.apache.oozie.util.ELEvaluator;
035import org.apache.oozie.util.XmlUtils;
036import org.jdom.Element;
037
038/**
039 * This class provide different evaluators required at different stages
040 */
041public class CoordELEvaluator {
042    public static final Integer MINUTE = 1;
043    public static final Integer HOUR = 60 * MINUTE;
044
045    /**
046     * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
047     * 1)
048     *
049     * @param conf : Configuration containing property variables
050     * @return configured ELEvaluator
051     */
052    public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
053        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
054        setConfigToEval(eval, conf);
055        return eval;
056    }
057
058    /**
059     * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
060     *
061     * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
062     * @param appInst : Application Instance related information such as Action creation Time
063     * @param conf :Configuration to substitute any variables
064     * @return configured ELEvaluator
065     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
066     */
067    public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
068            throws Exception {
069        return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
070    }
071
072    public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
073                                                         Configuration conf) throws Exception {
074        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
075        setConfigToEval(eval, conf);
076        SyncCoordDataset ds = getDSObject(event);
077        CoordELFunctions.configureEvaluator(eval, ds, appInst);
078        return eval;
079    }
080
081    public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
082                                                           HashMap<String, String> dataNameList) throws Exception {
083        ELEvaluator eval = createELEvaluatorForGroup(conf, group);
084        for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
085            String key = it.next();
086            String value = dataNameList.get(key);
087            eval.setVariable("oozie.dataname." + key, value);
088        }
089        return eval;
090    }
091
092    /**
093     * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
094     * should be resolved when all other data dependencies are met.
095     *
096     * @param actualTime : Action start time
097     * @param nominalTime : Action creation time
098     * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
099     * @param conf :Configuration to substitute any variables
100     * @return configured ELEvaluator
101     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
102     */
103    public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
104            throws Exception {
105        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
106        setConfigToEval(eval, conf);
107        SyncCoordDataset ds = getDSObject(dEvent);
108        SyncCoordAction appInst = new SyncCoordAction();
109        appInst.setNominalTime(nominalTime);
110        appInst.setActualTime(actualTime);
111        CoordELFunctions.configureEvaluator(eval, ds, appInst);
112        eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
113        return eval;
114    }
115
116    /**
117     * Create a SLA evaluator to be used during Materialization
118     * @param eAction
119     * @param coordAction
120     * @param conf
121     * @return
122     * @throws Exception
123     */
124    public static ELEvaluator createSLAEvaluator(Element eAction, CoordinatorActionBean coordAction, Configuration conf)
125            throws Exception {
126        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
127        setConfigToEval(eval, conf);
128        SyncCoordAction appInst = new SyncCoordAction();// TODO:
129        appInst.setNominalTime(coordAction.getNominalTime());
130        appInst.setActualTime(coordAction.getCreatedTime());
131        appInst.setActionId(coordAction.getId());
132        appInst.setName(eAction.getAttributeValue("name"));
133        CoordELFunctions.configureEvaluator(eval, null, appInst);
134
135        Element events = eAction.getChild("output-events", eAction.getNamespace());
136        if (events != null) {
137            for (Object obj : events.getChildren("data-out", eAction.getNamespace())) {
138                Element data = (Element) obj;
139                if (data.getChild("uris", data.getNamespace()) != null) {
140                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
141                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
142                    eval.setVariable(".dataout." + data.getAttributeValue("name"), uris);
143                }
144                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
145                    eval.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true");
146                }
147            }
148        }
149        return eval;
150    }
151
152    /**
153     * Create an Evaluator using conf and input/output-data (used for sla)
154     * @param conf
155     * @param group
156     * @param dataNameList
157     * @return
158     * @throws Exception
159     */
160    public static ELEvaluator createELEvaluatorForDataAndConf(Configuration conf, String group,
161            HashMap<String, String> dataNameList) throws Exception {
162        ELEvaluator eval = createELEvaluatorForDataEcho(conf, group, dataNameList);
163        setConfigToEval(eval, conf);
164        return eval;
165    }
166
167    /**
168     * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
169     *
170     * @param eJob : XML element for the application instance
171     * @param conf :Configuration to substitute any variables
172     * @return configured ELEvaluator
173     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
174     */
175    public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
176        ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
177        setConfigToEval(e, conf);
178        SyncCoordAction appInst = new SyncCoordAction();
179        String strNominalTime = eJob.getAttributeValue("action-nominal-time");
180        if (strNominalTime != null) {
181            appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
182            appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
183            appInst.setFrequency(eJob.getAttributeValue("frequency"));
184            appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
185            appInst.setActionId(actionId);
186            appInst.setName(eJob.getAttributeValue("name"));
187        }
188        String strActualTime = eJob.getAttributeValue("action-actual-time");
189        if (strActualTime != null) {
190            appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
191        }
192        CoordELFunctions.configureEvaluator(e, null, appInst);
193        Element events = eJob.getChild("input-events", eJob.getNamespace());
194        if (events != null) {
195            for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
196                if (data.getChild("uris", data.getNamespace()) != null) {
197                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
198                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
199                    e.setVariable(".datain." + data.getAttributeValue("name"), uris);
200                }
201                else {
202                }
203                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
204                    e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
205                    // check
206                    // null
207                }
208            }
209        }
210        events = eJob.getChild("output-events", eJob.getNamespace());
211        if (events != null) {
212            for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
213                if (data.getChild("uris", data.getNamespace()) != null) {
214                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
215                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
216                    e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
217                }
218                else {
219                }// TODO
220                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
221                    e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
222                    // check
223                    // null
224                }
225            }
226        }
227        return e;
228    }
229
230    /**
231     * Create a new Evaluator to resolve URI temple with time specific constant
232     *
233     * @param strDate : Date-time
234     * @return configured ELEvaluator
235     * @throws Exception If there is any date-time string in wrong format, the exception is thrown
236     */
237    public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
238        ELEvaluator eval = new ELEvaluator();
239        Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
240        // always???
241        date.setTime(DateUtils.parseDateOozieTZ(strDate));
242        eval.setVariable("YEAR", date.get(Calendar.YEAR));
243        eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
244        eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
245        eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
246        eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
247        return eval;
248    }
249
250    /**
251     * Create Dataset object using the Dataset XML information
252     *
253     * @param eData
254     * @return
255     * @throws Exception
256     */
257    private static SyncCoordDataset getDSObject(Element eData) throws Exception {
258        SyncCoordDataset ds = new SyncCoordDataset();
259        Element eDataset = eData.getChild("dataset", eData.getNamespace());
260        // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
261        Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
262        ds.setInitInstance(initInstance);
263        if (eDataset.getAttributeValue("frequency") != null) {
264            int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
265            ds.setFrequency(frequency);
266            ds.setType("SYNC");
267            if (eDataset.getAttributeValue("freq_timeunit") == null) {
268                throw new RuntimeException("No freq_timeunit defined in data set definition\n"
269                        + XmlUtils.prettyPrint(eDataset));
270            }
271            ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
272            if (eDataset.getAttributeValue("timezone") == null) {
273                throw new RuntimeException("No timezone defined in data set definition\n"
274                        + XmlUtils.prettyPrint(eDataset));
275            }
276            ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
277            if (eDataset.getAttributeValue("end_of_duration") == null) {
278                throw new RuntimeException("No end_of_duration defined in data set definition\n"
279                        + XmlUtils.prettyPrint(eDataset));
280            }
281            ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
282
283            Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
284            String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
285            ds.setDoneFlag(doneFlag);
286        }
287        else {
288            ds.setType("ASYNC");
289        }
290        String name = eDataset.getAttributeValue("name");
291        ds.setName(name);
292        // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
293        // eData.getNamespace()));
294        String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
295        ds.setUriTemplate(uriTemplate);
296        // ds.setTimeUnit(TimeUnit.MINUTES);
297        return ds;
298    }
299
300    /**
301     * Set all job configurations properties into evaluator.
302     *
303     * @param eval : Evaluator to set variables
304     * @param conf : configurations to set Evaluator
305     */
306    private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
307        for (Map.Entry<String, String> entry : conf) {
308            eval.setVariable(entry.getKey(), entry.getValue().trim());
309        }
310    }
311
312    /**
313     * make any one digit number to two digit string pre-appending a"0"
314     *
315     * @param num : number to make sting
316     * @return :String of length at least two digit.
317     */
318    private static String make2Digits(int num) {
319        String ret = "" + num;
320        if (num <= 9) {
321            ret = "0" + ret;
322        }
323        return ret;
324    }
325}