View Javadoc
1   /*******************************************************************************
2    *   Gisgraphy Project 
3    * 
4    *   This library is free software; you can redistribute it and/or
5    *   modify it under the terms of the GNU Lesser General Public
6    *   License as published by the Free Software Foundation; either
7    *   version 2.1 of the License, or (at your option) any later version.
8    * 
9    *   This library is distributed in the hope that it will be useful,
10   *   but WITHOUT ANY WARRANTY; without even the implied warranty of
11   *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12   *   Lesser General Public License for more details.
13   * 
14   *   You should have received a copy of the GNU Lesser General Public
15   *   License along with this library; if not, write to the Free Software
16   *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
17   * 
18   *  Copyright 2008  Gisgraphy project 
19   *  David Masclet <davidmasclet@gisgraphy.com>
20   *  
21   *  
22   *******************************************************************************/
23  package com.gisgraphy.importer;
24  
25  import java.io.File;
26  import java.util.ArrayList;
27  import java.util.List;
28  
29  import org.hibernate.FlushMode;
30  import org.hibernate.exception.ConstraintViolationException;
31  import org.slf4j.Logger;
32  import org.slf4j.LoggerFactory;
33  import org.springframework.beans.factory.annotation.Required;
34  
35  import com.gisgraphy.domain.geoloc.entity.City;
36  import com.gisgraphy.domain.geoloc.entity.GisFeature;
37  import com.gisgraphy.domain.repository.IGisFeatureDao;
38  import com.gisgraphy.domain.valueobject.GISSource;
39  import com.gisgraphy.domain.valueobject.NameValueDTO;
40  import com.gisgraphy.helper.GeolocHelper;
41  import com.vividsolutions.jts.geom.Geometry;
42  
43  /**
44   * Import the quattroshapes localities from an (pre-processed) Quattroshapes data file.
45   * The goal of this importer is to cross information between geonames and Quattroshapes 
46   * and add shape to cities
47   * 
48   * 
49   * @author <a href="mailto:david.masclet@gisgraphy.com">David Masclet</a>
50   */
51  public class QuattroshapesSimpleImporter extends AbstractSimpleImporterProcessor {
52  
53  	protected static final Logger logger = LoggerFactory.getLogger(QuattroshapesSimpleImporter.class);
54  
55  	protected IGisFeatureDao gisFeatureDao;
56  
57  
58  
59  	/* (non-Javadoc)
60  	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#flushAndClear()
61  	 */
62  	@Override
63  	protected void flushAndClear() {
64  		gisFeatureDao.flushAndClear();
65  	}
66  
67  
68  	/* (non-Javadoc)
69  	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#getFiles()
70  	 */
71  	@Override
72  	protected File[] getFiles() {
73  		return ImporterHelper.listCountryFilesToImport(importerConfig.getQuattroshapesDir());
74  	}
75  
76  	/* (non-Javadoc)
77  	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#getNumberOfColumns()
78  	 */
79  	@Override
80  	protected int getNumberOfColumns() {
81  		return 2;
82  	}
83  
84  	/* (non-Javadoc)
85  	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#processData(java.lang.String)
86  	 */
87  	@Override
88  	protected void processData(String line) throws ImporterException {
89  		String[] fields = line.split("\t");
90  		List<GisFeature> gisFeatures = new ArrayList<GisFeature>();
91  
92  		//
93  		// Line table has the following fields :
94  		// --------------------------------------------------- 
95  		//O : geonames id; 1 shape
96  		//
97  
98  		checkNumberOfColumn(fields);
99  
100 		//geonamesId
101 		if (!isEmptyField(fields, 0, false)) {
102 			String geonamesIds=fields[0].trim();
103 			String[] ids = geonamesIds.split(",");
104 			for (String geonamesId:ids){
105 				long geonamesIdAsLong;
106 				try {
107 					geonamesIdAsLong = Long.parseLong(geonamesId);
108 				} catch (NumberFormatException e) {
109 					logger.error("can not parse geonames id :"+geonamesId);
110 					//we don't want to parse when there is several ids (e.g:146390,146639) because it can cause error
111 					//todo if there is several ids search the rearest.
112 					continue;
113 				}
114 				GisFeature gisFeature = gisFeatureDao.getByFeatureId(geonamesIdAsLong);
115 				if (gisFeature != null){
116 					gisFeatures.add(gisFeature);
117 				} else {
118 					logger.warn("can not find gisfeature for geonames id "+geonamesId);
119 					continue;
120 				}
121 			}
122 
123 		} else {
124 			logger.warn("There is no geonames Id for "+dumpFields(fields));
125 			return;
126 		}
127 
128 		Geometry shape =null;
129 		if(!isEmptyField(fields, 1, false)){
130 			try {
131 				shape = (Geometry) GeolocHelper.convertFromHEXEWKBToGeometry(fields[1]);
132 			} catch (RuntimeException e) {
133 				logger.warn("can not parse shape for id "+fields[1]+" : "+e);
134 				return;
135 			}
136 		} else { 
137 			logger.warn("There is no shape for "+dumpFields(fields));
138 			return;
139 		}
140 
141 
142 
143 		for (GisFeature gisFeature:gisFeatures){
144 			if (gisFeatures.size()==1 || shapeContainsPoint(shape, gisFeature)){
145 				//if more than one feature match, we look at the one included in the shape
146 				gisFeature.setShape(shape);
147 				if (gisFeature.getSource()==GISSource.GEONAMES){
148 					gisFeature.setSource(GISSource.GEONAMES_QUATTRO);
149 				} else if (gisFeature.getSource() ==GISSource.GEONAMES_OSM){
150 					gisFeature.setSource(GISSource.GEONAMES_OSM_QUATTRO);
151 				}
152 				if (gisFeature instanceof City){
153 					((City) gisFeature).setMunicipality(true);//force to be a municipality.
154 				}
155 
156 				try {
157 					savecity(gisFeature);
158 				} catch (ConstraintViolationException e) {
159 					logger.error("Can not save "+dumpFields(fields)+"(ConstraintViolationException) we continue anyway but you should consider this",e);
160 				}catch (Exception e) {
161 					logger.error("Can not save "+dumpFields(fields)+" we continue anyway but you should consider this",e);
162 				}
163 			}
164 		}
165 
166 
167 	}
168 
169 
170 	private boolean shapeContainsPoint(Geometry shape, GisFeature gisFeature) {
171 				try {
172 					return gisFeature.getLocation().within(shape);
173 				} catch (Exception e) {
174 					logger.error("can not determine if a shape contains point for "+gisFeature);
175 					return false;
176 				}
177 	}
178 
179 	/**
180 	 * @param fields
181 	 *                The array to process
182 	 * @return a string which represent a human readable string of the Array but without shape because it is useless in logs
183 	 */
184 	protected static String dumpFields(String[] fields) {
185 		String result = "[";
186 		for (int i=0;i<fields.length;i++) {
187 			if (i==1){
188 				result= result+"THE_SHAPE;";
189 			}else {
190 				result = result + fields[i] + ";";
191 			}
192 		}
193 		return result + "]";
194 	}
195 
196 
197 	void savecity(GisFeature gisFeature) {
198 		if (gisFeature!=null){
199 			gisFeatureDao.save(gisFeature);
200 		}
201 	}
202 
203 
204 
205 	/* (non-Javadoc)
206 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#shouldBeSkiped()
207 	 */
208 	@Override
209 	public boolean shouldBeSkipped() {
210 		return !importerConfig.isQuattroshapesImporterEnabled() || !importerConfig.isGeonamesImporterEnabled();
211 	}
212 
213 
214 
215 
216 	/* (non-Javadoc)
217 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#setCommitFlushMode()
218 	 */
219 	@Override
220 	protected void setCommitFlushMode() {
221 		this.gisFeatureDao.setFlushMode(FlushMode.COMMIT);
222 	}
223 
224 	/* (non-Javadoc)
225 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#shouldIgnoreComments()
226 	 */
227 	@Override
228 	protected boolean shouldIgnoreComments() {
229 		return true;
230 	}
231 
232 	/* (non-Javadoc)
233 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#shouldIgnoreFirstLine()
234 	 */
235 	@Override
236 	protected boolean shouldIgnoreFirstLine() {
237 		return false;
238 	}
239 
240 	/* (non-Javadoc)
241 	 * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#rollback()
242 	 */
243 	public List<NameValueDTO<Integer>> rollback() {
244 		List<NameValueDTO<Integer>> deletedObjectInfo = new ArrayList<NameValueDTO<Integer>>();
245 		logger.info("reseting quattroshapes");
246 		//TODO only cities that have source openstreetmap
247 		deletedObjectInfo
248 		.add(new NameValueDTO<Integer>(City.class.getSimpleName(), 0));
249 		resetStatus();
250 		return deletedObjectInfo;
251 	}
252 
253 
254 	@Override
255 	//TODO test
256 	protected void tearDown() {
257 		super.tearDown();
258 		String savedMessage = this.statusMessage;
259 		try {
260 			this.statusMessage = internationalisationService.getString("import.fulltext.optimize");
261 		} finally {
262 			// we restore message in case of error
263 			this.statusMessage = savedMessage;
264 		}
265 	}
266 
267 
268 
269 	@Required
270 	public void setGisFeatureDao(IGisFeatureDao gisFeatureDao) {
271 		this.gisFeatureDao = gisFeatureDao;
272 	}
273 
274 
275 }