View Javadoc
1   /*******************************************************************************
2    *   Gisgraphy Project 
3    * 
4    *   This library is free software; you can redistribute it and/or
5    *   modify it under the terms of the GNU Lesser General Public
6    *   License as published by the Free Software Foundation; either
7    *   version 2.1 of the License, or (at your option) any later version.
8    * 
9    *   This library is distributed in the hope that it will be useful,
10   *   but WITHOUT ANY WARRANTY; without even the implied warranty of
11   *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12   *   Lesser General Public License for more details.
13   * 
14   *   You should have received a copy of the GNU Lesser General Public
15   *   License along with this library; if not, write to the Free Software
16   *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
17   * 
18   *  Copyright 2008  Gisgraphy project 
19   *  David Masclet <davidmasclet@gisgraphy.com>
20   *  
21   *  
22   *******************************************************************************/
23  package com.gisgraphy.importer;
24  
25  import java.io.File;
26  import java.util.ArrayList;
27  import java.util.List;
28  
29  import org.hibernate.FlushMode;
30  import org.slf4j.Logger;
31  import org.slf4j.LoggerFactory;
32  import org.springframework.beans.factory.annotation.Required;
33  
34  import com.gisgraphy.domain.geoloc.entity.AlternateName;
35  import com.gisgraphy.domain.geoloc.entity.GisFeature;
36  import com.gisgraphy.domain.geoloc.entity.ZipCode;
37  import com.gisgraphy.domain.geoloc.entity.ZipCodesAware;
38  import com.gisgraphy.domain.repository.IAdmDao;
39  import com.gisgraphy.domain.repository.IAlternateNameDao;
40  import com.gisgraphy.domain.repository.ICityDao;
41  import com.gisgraphy.domain.repository.IGisFeatureDao;
42  import com.gisgraphy.domain.repository.ISolRSynchroniser;
43  import com.gisgraphy.domain.valueobject.AlternateNameSource;
44  import com.gisgraphy.domain.valueobject.NameValueDTO;
45  import com.gisgraphy.fulltext.spell.ISpellCheckerIndexer;
46  
47  /**
48   * Import the Alternate names.
49   * 
50   * @author <a href="mailto:david.masclet@gisgraphy.com">David Masclet</a>
51   */
52  public class GeonamesAlternateNamesSimpleImporter extends AbstractSimpleImporterProcessor {
53  	
54  	protected static final Logger logger = LoggerFactory.getLogger(GeonamesAlternateNamesSimpleImporter.class);
55  
56  	protected IGisFeatureDao gisFeatureDao;
57  
58      protected ICityDao cityDao;
59      
60      protected IAdmDao admDao;
61  
62      protected IAlternateNameDao alternateNameDao;
63  
64      protected ISolRSynchroniser solRSynchroniser;
65  
66      private static final Long defaultFeatureId = Long.valueOf(-1);
67      
68      protected ISpellCheckerIndexer spellCheckerIndexer;
69  
70  
71      /**
72       * Default constructor
73       */
74      public GeonamesAlternateNamesSimpleImporter() {
75  	super();
76      }
77  
78      /*
79       * (non-Javadoc)
80       * 
81       * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#processData(java.lang.String)
82       */
83      @Override
84      protected void processData(String line) {
85  	String[] fields = line.split("\t");
86  
87  	/*
88  	 * line table has the following fields :
89  	 * ----------------------------------------- 0 : alternateNameId : 1 :
90  	 * geonameid : 2 : isolanguage : iso 639-2 or 3 or or 'post' 3 :
91  	 * alternate name 4 : isPreferredName 5 : isShortName
92  	 */
93  
94  	AlternateName alternateName = null;
95  	// we don't check length because some fields like preferredname or
96  	// shortname are not mandatory
97  	// checkNumberOfColumn(fields);
98  	if (!isEmptyField(fields, 0, true)) {
99  	    // check if need to be inserted or updated
100 	    // alternateName = this.alternateNameDao.get(new Long(fields[0]));
101 	    // if (alternateName == null) {
102 	    alternateName = new AlternateName();
103 	    alternateName.setAlternateNameId(new Integer(fields[0]));
104 	    // }
105 
106 	}
107 
108 	GisFeature gisFeature = null;
109 
110 	Long gisFeatureId = defaultFeatureId;
111 	if (!isEmptyField(fields, 1, true)) {
112 	    try {
113 		gisFeatureId = new Long(fields[1]);
114 	    } catch (NumberFormatException e) {
115 		logger.warn("The featureId " + fields[1] + " is not a number");
116 		return;
117 	    }
118 	    // get the features
119 	   // to improve performance we first search in cities then adm and finally in all features
120 	    gisFeature = cityDao.getByFeatureId(gisFeatureId);
121 	    if (gisFeature == null){
122 		gisFeature = this.admDao.getByFeatureId(gisFeatureId);
123 	    }
124 	    if (gisFeature == null){
125 		gisFeature = this.gisFeatureDao.getByFeatureId(gisFeatureId);
126 	    }
127 	    if (gisFeature == null) {
128 		return;
129 	    }
130 
131 	}
132 
133 	alternateName.setGisFeature(gisFeature);
134 	if (isAnUnWantedLanguageField(fields[2])) {
135 	    return;
136 	}
137 
138 	// it is not an empty field =>will save the alternatename
139 
140 	// set name and sources
141 	if (!isEmptyField(fields, 3, true)) {
142 	    alternateName.setName(fields[3].trim());
143 	}
144 
145 	alternateName.setSource(AlternateNameSource.ALTERNATENAMES_FILE);
146 
147 	// if it is a post=>update zipCode
148 	if (fields[2].equals("post")) {
149 	    // it is a zip code :set and save gisfeature
150 	    if (!isEmptyField(fields, 3, false)) {
151 		// if it is a city we update the city
152 		    if (!(gisFeature instanceof ZipCodesAware)) {
153 		    	logger.error("We've got a zipCode for a feature ("+gisFeature+") but "+gisFeature.getClass().getSimpleName()+
154 						" is not 'zipCodes aware'," +
155 						" that mean that zipcode is added but the field" +
156 						" will not be added in the feed of the web service");
157 		    } 
158 		    gisFeature.addZipCode(new ZipCode(fields[3]));
159 			// Hibernate will save the feature according to his
160 			// class even if it is cast in an other class
161 			this.gisFeatureDao.save(gisFeature);
162 			return;
163 	    } else {
164 		logger.warn("could not set ZipCode for GisFeature["
165 			+ gisFeatureId + "] because zipCode is null");
166 		return;
167 	    }
168 	} else {
169 	    // it is probably a language field;
170 	    alternateName.setLanguage(fields[2]);
171 	}
172 
173 	// preferred name
174 	if (!isEmptyField(fields, 4, false)) {
175 	    if (fields[4].equals("1")) {
176 		alternateName.setPreferredName(true);
177 	    } else {
178 		alternateName.setPreferredName(false);
179 	    }
180 
181 	}
182 
183 	// short name
184 	if (!isEmptyField(fields, 5, false)) {
185 	    if (fields[5].equals("1")) {
186 		alternateName.setShortName(true);
187 	    } else {
188 		alternateName.setShortName(false);
189 	    }
190 	}
191 
192 	/*List<AlternateName> alternateNames = gisFeature.getAlternateNames();
193 	if (alternateNames == null) {
194 	    alternateNames = new ArrayList<AlternateName>();
195 	}
196 	alternateNames.add(alternateName);*/
197 	gisFeature.addAlternateName(alternateName);
198 
199 	this.gisFeatureDao.save(gisFeature);
200 
201     }
202 
203     private boolean isAnUnWantedLanguageField(String languageField) {
204 	boolean unWanted = false;
205 	// TODO v2 option : || languageField.equals("icao") ||
206 	// languageField.equals("iata")
207 	if (languageField.equals("fr-1793")) {
208 	    unWanted = true;
209 	}
210 	return unWanted;
211 
212     }
213 
214     /*
215      * (non-Javadoc)
216      * 
217      * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#shouldIgnoreFirstLine()
218      */
219     @Override
220     protected boolean shouldIgnoreFirstLine() {
221 	return false;
222     }
223 
224     /*
225      * (non-Javadoc)
226      * 
227      * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#shouldIgnoreComments()
228      */
229     @Override
230     protected boolean shouldIgnoreComments() {
231 	return true;
232     }
233 
234     /*
235      * (non-Javadoc)
236      * 
237      * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#setCommitFlushMode()
238      */
239     @Override
240     protected void setCommitFlushMode() {
241 	this.gisFeatureDao.setFlushMode(FlushMode.COMMIT);
242 	this.cityDao.setFlushMode(FlushMode.COMMIT);
243 	this.alternateNameDao.setFlushMode(FlushMode.COMMIT);
244     }
245 
246     @Override
247     protected void flushAndClear() {
248 	this.gisFeatureDao.flushAndClear();
249 	this.cityDao.flushAndClear();
250 	this.alternateNameDao.flushAndClear();
251     }
252 
253     /*
254      * (non-Javadoc)
255      * 
256      * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#getNumberOfColumns()
257      */
258     @Override
259     protected int getNumberOfColumns() {
260 	return 6;
261     }
262 
263     /*
264      * (non-Javadoc)
265      * 
266      * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#tearDown()
267      */
268     @Override
269     protected void tearDown() {
270 	super.tearDown();
271 	if (!solRSynchroniser.commit()){
272 	    logger.warn("The commit in tearDown of "+this.getClass().getSimpleName()+" has failed, the uncommitted changes will be commited with the auto commit of solr in few minuts");
273 	}
274 	spellCheckerIndexer.buildAllIndex();
275 	solRSynchroniser.optimize();
276     }
277 
278     /**
279      * @param alternateNameDao
280      *                The alternateDao to set
281      */
282     @Required
283     public void setAlternateNameDao(IAlternateNameDao alternateNameDao) {
284 	this.alternateNameDao = alternateNameDao;
285     }
286 
287     /**
288      * @param gisFeatureDao
289      *                The GisFeatureDao to set
290      */
291     @Required
292     public void setGisFeatureDao(IGisFeatureDao gisFeatureDao) {
293 	this.gisFeatureDao = gisFeatureDao;
294     }
295 
296     /**
297      * @param cityDao
298      *                The cityDao to set
299      */
300     @Required
301     public void setCityDao(ICityDao cityDao) {
302 	this.cityDao = cityDao;
303     }
304 
305     
306     /**
307      * @param admDao the admDao to set
308      */
309     @Required
310     public void setAdmDao(IAdmDao admDao) {
311         this.admDao = admDao;
312     }
313 
314     @Override
315     public boolean shouldBeSkipped() {
316         if (importerConfig.isImportGisFeatureEmbededAlternateNames() || !importerConfig.isGeonamesImporterEnabled()){
317             return true ;
318         }
319         return false;
320     }
321     
322     /*
323      * (non-Javadoc)
324      * 
325      * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#getFiles()
326      */
327     @Override
328     protected File[] getFiles() {
329 	if (importerConfig.isImportGisFeatureEmbededAlternateNames()) {
330 	    logger
331 		    .info("ImportGisFeatureEmbededAlternateNames = true, we do not import alternatenames from "
332 			    + importerConfig.getAlternateNamesFileName());
333 	    return new File[0];
334 	}
335 	File[] files = new File[1];
336 	files[0] = new File(importerConfig.getGeonamesDir()
337 		+ importerConfig.getAlternateNameFeaturesFileName());
338 	return files;
339     }
340 
341     /**
342      * @param solRSynchroniser
343      *                the solRSynchroniser to set
344      */
345     @Required
346     public void setSolRSynchroniser(ISolRSynchroniser solRSynchroniser) {
347 	this.solRSynchroniser = solRSynchroniser;
348     }
349     
350     /**
351      * @param spellCheckerIndexer the spellCheckerIndexer to set
352      */
353     public void setSpellCheckerIndexer(ISpellCheckerIndexer spellCheckerIndexer) {
354         this.spellCheckerIndexer = spellCheckerIndexer;
355     }
356 
357     /*
358      * (non-Javadoc)
359      * 
360      * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#rollback()
361      */
362     public List<NameValueDTO<Integer>> rollback() {
363 	List<NameValueDTO<Integer>> deletedObjectInfo = new ArrayList<NameValueDTO<Integer>>();
364 	logger.info("deleting alternateNames...");
365 	int deletedAnames = alternateNameDao.deleteAll();
366 	if (deletedAnames != 0) {
367 	    deletedObjectInfo.add(new NameValueDTO<Integer>(GisFeature.class
368 		    .getSimpleName(), deletedAnames));
369 	}
370 	logger.info(deletedAnames + " alternateNames have been deleted");
371 	resetStatus();
372 	return deletedObjectInfo;
373     }
374 
375 }