View Javadoc
1   /*******************************************************************************
2    *   Gisgraphy Project 
3    * 
4    *   This library is free software; you can redistribute it and/or
5    *   modify it under the terms of the GNU Lesser General Public
6    *   License as published by the Free Software Foundation; either
7    *   version 2.1 of the License, or (at your option) any later version.
8    * 
9    *   This library is distributed in the hope that it will be useful,
10   *   but WITHOUT ANY WARRANTY; without even the implied warranty of
11   *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12   *   Lesser General Public License for more details.
13   * 
14   *   You should have received a copy of the GNU Lesser General Public
15   *   License along with this library; if not, write to the Free Software
16   *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
17   * 
18   *  Copyright 2008  Gisgraphy project 
19   *  David Masclet <davidmasclet@gisgraphy.com>
20   *  
21   *  
22   *******************************************************************************/
23  package com.gisgraphy.importer;
24  
25  import static com.gisgraphy.street.HouseNumberUtil.normalizeNumber;
26  
27  import java.io.File;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.List;
31  import java.util.regex.Matcher;
32  import java.util.regex.Pattern;
33  
34  import org.hibernate.FlushMode;
35  import org.slf4j.Logger;
36  import org.slf4j.LoggerFactory;
37  import org.springframework.beans.factory.annotation.Required;
38  
39  import com.gisgraphy.domain.geoloc.entity.HouseNumber;
40  import com.gisgraphy.domain.geoloc.entity.OpenStreetMap;
41  import com.gisgraphy.domain.repository.IOpenStreetMapDao;
42  import com.gisgraphy.domain.repository.ISolRSynchroniser;
43  import com.gisgraphy.domain.repository.IhouseNumberDao;
44  import com.gisgraphy.domain.valueobject.HouseNumberType;
45  import com.gisgraphy.domain.valueobject.NameValueDTO;
46  import com.gisgraphy.domain.valueobject.Output;
47  import com.gisgraphy.domain.valueobject.Output.OutputStyle;
48  import com.gisgraphy.domain.valueobject.Pagination;
49  import com.gisgraphy.fulltext.FullTextSearchEngine;
50  import com.gisgraphy.fulltext.FulltextQuery;
51  import com.gisgraphy.fulltext.FulltextResultsDto;
52  import com.gisgraphy.fulltext.IFullTextSearchEngine;
53  import com.gisgraphy.fulltext.SolrResponseDto;
54  import com.gisgraphy.helper.GeolocHelper;
55  import com.gisgraphy.importer.dto.AddressInclusion;
56  import com.gisgraphy.importer.dto.AssociatedStreetHouseNumber;
57  import com.gisgraphy.importer.dto.AssociatedStreetMember;
58  import com.gisgraphy.importer.dto.InterpolationHouseNumber;
59  import com.gisgraphy.importer.dto.InterpolationMember;
60  import com.gisgraphy.importer.dto.InterpolationType;
61  import com.gisgraphy.importer.dto.NodeHouseNumber;
62  import com.gisgraphy.service.ServiceException;
63  import com.vividsolutions.jts.geom.Point;
64  
65  /**
66   * Import the street from an (pre-processed) openStreet map data file .
67   * 
68   * @author <a href="mailto:david.masclet@gisgraphy.com">David Masclet</a>
69   */
70  public class OpenStreetMapHouseNumberSimpleImporter extends AbstractSimpleImporterProcessor {
71  
72  
73  	protected static final Logger logger = LoggerFactory.getLogger(OpenStreetMapHouseNumberSimpleImporter.class);
74  
75  	protected IOpenStreetMapDao openStreetMapDao;
76  
77  	protected IhouseNumberDao houseNumberDao;
78  
79  	protected ISolRSynchroniser solRSynchroniser;
80  
81  	protected IFullTextSearchEngine fullTextSearchEngine;
82  
83  	private static final String ASSOCIATED_HOUSE_NUMBER_REGEXP = "([0-9]+)___([^_]*)___((?:(?!___).)*)___((?:(?!___).)*)___([NW])___([^_]*)(?:___)?";
84  
85  	private static final String INTERPOLATION_HOUSE_NUMBER_REGEXP = "([0-9]+)___([0-9])___((?:(?!___).)+)*___((?:(?!___).)+)*___((?:(?!___).)+)*(?:___)?";
86  
87  	private static final Pattern ASSOCIATED_HOUSE_NUMBER_PATTERN = Pattern.compile(ASSOCIATED_HOUSE_NUMBER_REGEXP, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
88  
89  	private static final Pattern INTERPOLATION_HOUSE_NUMBER_PATTERN = Pattern.compile(INTERPOLATION_HOUSE_NUMBER_REGEXP, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
90  	
91  	protected final static Output MEDIUM_OUTPUT = Output.withDefaultFormat().withStyle(OutputStyle.MEDIUM);
92  
93  	protected static final double SEARCH_DISTANCE = 6000;
94  
95  	
96  	/*
97  	 * (non-Javadoc)
98  	 * 
99  	 * @see
100 	 * com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#flushAndClear
101 	 * ()
102 	 */
103 	@Override
104 	protected void flushAndClear() {
105 		openStreetMapDao.flushAndClear();
106 		houseNumberDao.flushAndClear();
107 
108 	}
109 
110 	@Override
111 	protected void setup() {
112 		//temporary disable logging when importing
113 		FullTextSearchEngine.disableLogging=true;
114 		super.setup();
115 	}
116 
117 	/*
118 	 * (non-Javadoc)
119 	 * 
120 	 * @see
121 	 * com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#getFiles()
122 	 */
123 	@Override
124 	protected File[] getFiles() {
125 		return ImporterHelper.listCountryFilesToImport(importerConfig.getOpenStreetMapHouseNumberDir());
126 	}
127 
128 	/*
129 	 * (non-Javadoc)
130 	 * 
131 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#
132 	 * getNumberOfColumns()
133 	 */
134 	@Override
135 	protected int getNumberOfColumns() {
136 		return 9;
137 	}
138 
139 	protected AssociatedStreetHouseNumber parseAssociatedStreetHouseNumber(String line) {
140 		/*
141 		 * A 1264114 "{"
142 		 * "47129758___0101000020E61000005CCBD3C231E76240AA6514FE5BF440C0___Bowral Street___Bowral Street___W___street"
143 		 * ", ""84623507
144 		 * ___0101000020E6100000546690CC36E76240A417D5545AF440C0___71___Bowral
145 		 * Street___W___house""}"
146 		 */
147 		if (line == null || "".equals(line.trim())) {
148 			return null;
149 		}
150 		String[] fields = line.split("\t");
151 		if (fields.length != 3) {
152 			logger.warn("wrong number of fields for line " + line + " expected 3 but was " + fields.length);
153 			return null;
154 		}
155 		if (!"A".equals(fields[0])) {
156 			logger.warn("wrong house Number Type for line " + line + " expected 'A' but was " + fields[0]);
157 			return null;
158 		}
159 		AssociatedStreetHouseNumber houseNumber = new AssociatedStreetHouseNumber();
160 		if (!isEmptyField(fields, 1, false)) {
161 			houseNumber.setRelationID(fields[1].trim());
162 		}
163 		if (!isEmptyField(fields, 2, false)) {
164 			Matcher matcher = ASSOCIATED_HOUSE_NUMBER_PATTERN.matcher(fields[2].trim());
165 			int i = 0;
166 			while (matcher.find()) {
167 				AssociatedStreetMember member = new AssociatedStreetMember();
168 				if (matcher.groupCount() != 6) {
169 					logger.warn("wrong number of fields for AssociatedStreetMember no " + i + "for line " + line);
170 					continue;
171 				}
172 				member.setId(matcher.group(1));
173 				Point point;
174 				try {
175 					point = (Point) GeolocHelper.convertFromHEXEWKBToGeometry(matcher.group(2));
176 				} catch (Exception e) {
177 					logger.warn(e.getMessage());
178 					return null;
179 				}
180 				if (point == null) {
181 					logger.warn("wrong location for AssociatedStreetMember for point n" + i + "for line " + line);
182 					continue;
183 				}
184 				member.setLocation(point);
185 				String role = matcher.group(6);
186 				member.setRole(role);
187 				member.setHouseNumber(matcher.group(3));
188 				member.setStreetName(matcher.group(4));
189 				member.setType(matcher.group(5));
190 
191 				houseNumber.addMember(member);
192 				i++;
193 			}
194 
195 		} else {
196 			return null;
197 		}
198 		return houseNumber;
199 	}
200 
201 	protected InterpolationHouseNumber parseInterpolationHouseNumber(String line) {
202 		/*
203 		 * I	168365171	1796478450___0___0101000020E61000009A023EE4525350C0959C137B682F38C0______;
204 		 * 1366275082___1___0101000020E610000068661CD94B5350C0B055270C6F2F38C0______;
205 		 * 1796453793___2___0101000020E610000038691A144D5350C023ADE75A6A2F38C0___600___;
206 		 * 1796453794___3___0101000020E6100000F38F6390605350C028A6666A6D2F38C0___698___		even
207 		 */
208 		if (line == null || "".equals(line.trim())) {
209 			return null;
210 		}
211 		String[] fields = line.split("\t");
212 		if (fields.length < 5 || fields.length > 6) {
213 			logger.warn("wrong number of fields for line " + line + " expected 5/6 but was " + fields.length);
214 			return null;
215 		}
216 		if (!"I".equals(fields[0])) {
217 			logger.warn("wrong house Number Type for line " + line + " expected 'A' but was " + fields[0]);
218 			return null;
219 		}
220 		InterpolationHouseNumber houseNumber = new InterpolationHouseNumber();
221 		if (!isEmptyField(fields, 1, false)) {
222 			houseNumber.setWayId(fields[1].trim());
223 		}
224 		if (!isEmptyField(fields, 4, false)) {
225 			
226 			try {
227 				houseNumber.setInterpolationType(InterpolationType.valueOf(fields[4].trim().toLowerCase()));
228 			} catch (Exception e) {
229 				//ignore
230 			}
231 		}
232 		if (!isEmptyField(fields, 3, false)) {
233 			houseNumber.setStreetName(fields[3].trim());
234 		}
235 		if (!isEmptyField(fields, 5, false)) {
236 			
237 			try {
238 				houseNumber.setAddressInclusion(AddressInclusion.valueOf(fields[5].trim().toLowerCase()));
239 			} catch (Exception e) {
240 				//ignore
241 			}
242 		}
243 
244 		if (!isEmptyField(fields, 2, false)) {
245 			Matcher matcher = INTERPOLATION_HOUSE_NUMBER_PATTERN.matcher(fields[2].trim());
246 			int i = 0;
247 			while (matcher.find()) {
248 				InterpolationMember member = new InterpolationMember();
249 				if (matcher.groupCount() != 5) {
250 					logger.warn("wrong number of fields for InterpolationMember n" + i + "for line " + line);
251 					continue;
252 				}
253 				member.setId(matcher.group(1));
254 				// seqId
255 				String seqIdAsString = matcher.group(2);
256 				int seqId = 0;
257 				try {
258 					seqId = Integer.parseInt(seqIdAsString);
259 				} catch (NumberFormatException e) {
260 					logger.warn("can not convert sequence id "+seqIdAsString+" to integer");
261 					continue;
262 				}
263 				member.setSequenceId(seqId);
264 				// location
265 				Point point;
266 				try {
267 					point = (Point) GeolocHelper.convertFromHEXEWKBToGeometry(matcher.group(3));
268 				} catch (Exception e) {
269 					logger.warn(e.getMessage());
270 					return null;
271 				}
272 				if (point == null) {
273 					logger.warn("wrong location for InterpolationMember point n" + i + "for line " + line);
274 					continue;
275 				}
276 				member.setLocation(point);
277 
278 				member.setHouseNumber(matcher.group(4));
279 				member.setStreetname(matcher.group(5));
280 				houseNumber.addMember(member);
281 				i++;
282 			}
283 			Collections.sort(houseNumber.getMembers());
284 		} else {
285 			return null;
286 		}
287 		
288 		return houseNumber;
289 	}
290 
291 	protected NodeHouseNumber parseNodeHouseNumber(String line) {
292 		//N	1053493828	0101000020E610000060910486D17250C05D4B6D4ECA753CC0	75	Sandwichs La Estrellita	Estanislao Maldones
293 		if (line == null || "".equals(line.trim())) {
294 			return null;
295 		}
296 		String[] fields = line.split("\t");
297 		if (fields.length < 4 ) {
298 			logger.warn("wrong number of fields for line " + line + " expected 4 but was " + fields.length);
299 			return null;
300 		}
301 		if (!"N".equals(fields[0]) && !"W".equals(fields[0])) {
302 			logger.warn("wrong house Number Type for line " + line + " expected 'N' or 'w' but was " + fields[0]);
303 			return null;
304 		}
305 		NodeHouseNumber node = new NodeHouseNumber();
306 		if (!isEmptyField(fields, 1, false)) {
307 			node.setNodeId(fields[1].trim());
308 		}
309 		if (!isEmptyField(fields, 2, false)) {
310 			Point point;
311 			try {
312 				point = (Point) GeolocHelper.convertFromHEXEWKBToGeometry(fields[2].trim());
313 			} catch (Exception e) {
314 				logger.warn(e.getMessage());
315 				return null;
316 			}
317 			if (point == null) {
318 				logger.warn("wrong location for NodeHouseNumber for point for line " + line);
319 				return null;
320 			} else {
321 				node.setLocation(point);
322 			}
323 		}
324 		if (!isEmptyField(fields, 3, false)) {
325 			node.setHouseNumber(fields[3].trim());
326 		}
327 		if (!isEmptyField(fields, 4, false)) {
328 			node.setName(fields[4].trim());
329 		}
330 		if (!isEmptyField(fields, 5, false)) {
331 			node.setStreetName(fields[5].trim());
332 		}
333 		return node;
334 	}
335 	
336 
337 	/*
338 	 * (non-Javadoc)
339 	 * 
340 	 * @see
341 	 * com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#processData
342 	 * (java.lang.String)
343 	 */
344 	@Override
345 	protected void processData(String line) throws ImporterException {
346 		if (line==null || "".equals(line.trim())){
347 			return;
348 		}
349 		
350 		if (line.startsWith("A")){
351 			AssociatedStreetHouseNumber house = parseAssociatedStreetHouseNumber(line);
352 			if (house!=null){
353 				processAssociatedStreet(house);
354 			}
355 		} else if (line.startsWith("N")){
356 			NodeHouseNumber house = parseNodeHouseNumber(line);
357 			if (house!=null){
358 				processNodeHouseNumber(house);
359 			}
360 		}else if (line.startsWith("W")){
361 			NodeHouseNumber house = parseNodeHouseNumber(line);
362 			if (house!=null){
363 				processNodeHouseNumber(house);
364 			}
365 		}  else if (line.startsWith("I")) {
366 			InterpolationHouseNumber house = parseInterpolationHouseNumber(line);
367 			if(house==null){
368 				return;
369 			}
370 			List<InterpolationMember> members = house.getMembers();
371 			if (members.size() <= 1) {
372 				//we can not interpolate if there is less than 2 points
373 				logger.warn("can not interpolate if there is less than two points for " + line);
374 				return;
375 			}
376 			OpenStreetMap osm = null;
377 			if (house.getStreetName() != null && !"".equals(house.getStreetName().trim()) && !"\"\"".equals(house.getStreetName().trim())) {
378 				osm = findNearestStreet(house.getStreetName(), members.get(0).getLocation());
379 				if (osm == null) {
380 					logger.warn("can not find street for name "+house.getStreetName()+", position :"+ members.get(0).getLocation());
381 					return;// we don't know which street to add the numbers
382 				}
383 			} else {
384 				return;
385 			}
386 			List<HouseNumber> houseNumbers = processInterpolationHouseNumber(house);
387 			if (houseNumbers.size()!=0){
388 				osm.addHouseNumbers(houseNumbers);
389 				openStreetMapDao.save(osm);
390 			}
391 		} else {
392 			logger.warn("unknow node type for line " + line);
393 		}
394 
395 	}
396 
397 	protected void processAssociatedStreet(AssociatedStreetHouseNumber house) {
398 		if (house==null){
399 			return;
400 		}
401 		List<AssociatedStreetMember> streetMembers = house.getStreetMembers();
402 		List<AssociatedStreetMember> houseMembers = house.getHouseMembers();
403 		if (houseMembers.size()==0 ){
404 			//no streets or no house
405 			logger.warn("there is no member for associated street "+house);
406 			return;
407 		} 
408 		if (streetMembers.size()==0){
409 			//treet as node, it is often the case when associated with a relation and our script don't manage it so we link it here
410 			if (houseMembers!=null){
411 				String streetname = null;
412 				boolean allHouseHaveTheSameStreetName = true;
413 				OpenStreetMap street=null;
414 				for (AssociatedStreetMember houseMember : houseMembers){
415 					if (streetname==null && houseMember.getStreetName()!=null){
416 						streetname=houseMember.getStreetName();
417 						street = findNearestStreet(houseMember.getStreetName(),houseMember.getLocation());
418 						continue;
419 					}else {
420 						if (!houseMember.getStreetName().equals(streetname)){
421 							allHouseHaveTheSameStreetName=false;
422 							street=null;
423 							break;
424 						}
425 					}
426 				}
427 				
428 			for (AssociatedStreetMember houseMember : houseMembers){
429 				if (houseMember.getStreetName()!=null && !"".equals(houseMember.getStreetName().trim()) && houseMember.getLocation()!=null){
430 					if (!allHouseHaveTheSameStreetName) {//we have to find for each one
431 						street = findNearestStreet(houseMember.getStreetName(),houseMember.getLocation());
432 					}
433 					if (street!=null){
434 						//Long openstreetmapId = street.getOpenstreetmap_id();
435 						//OpenStreetMap osm = openStreetMapDao.getByOpenStreetMapId(openstreetmapId);
436 						HouseNumber houseNumber = buildHouseNumberFromAssociatedHouseNumber(houseMember);
437 						if (houseNumber!=null){
438 							street.addHouseNumber(houseNumber);
439 							openStreetMapDao.save(street);
440 						}
441 					} else {
442 						logger.warn("can not find associated street for name "+houseMember.getStreetName()+", position :"+ houseMember.getLocation());
443 					}
444 				}
445 			}
446 			}
447 		}
448 		else if (streetMembers.size()==1){
449 			AssociatedStreetMember associatedStreetMember = streetMembers.get(0);
450 			if (associatedStreetMember.getId()==null){
451 				logger.warn("associated street "+associatedStreetMember+" has no id");
452 				return;
453 			}
454 			Long idAsLong = null;
455 			try {
456 				idAsLong = Long.valueOf(associatedStreetMember.getId());
457 			} catch (NumberFormatException e) {
458 				logger.warn(idAsLong+" is not a valid id for associated street");
459 				return;
460 			}
461 			OpenStreetMap associatedStreet = openStreetMapDao.getByOpenStreetMapId(idAsLong);
462 			if (associatedStreet==null){
463 				logger.warn("no street can be found for associated street for id "+idAsLong);
464 				return;
465 			}
466 			for (AssociatedStreetMember houseMember : houseMembers){
467 				HouseNumber houseNumber = buildHouseNumberFromAssociatedHouseNumber(houseMember);
468 				if (houseNumber!=null){
469 					associatedStreet.addHouseNumber(houseNumber);
470 				}
471 			}
472 			openStreetMapDao.save(associatedStreet);
473 		}
474 		else if (streetMembers.size()>1){
475 			//for each house, search the nearest street
476 			//getStreetIds
477 			List<Long>  streetIds = new ArrayList<Long>();
478 			for (AssociatedStreetMember street : streetMembers){
479 				Long id;
480 				try {
481 					id = Long.valueOf(street.getId());
482 					streetIds.add(id);
483 				} catch (NumberFormatException e) {
484 					logger.warn(street+" has no id");
485 				}
486 			}
487 			for (AssociatedStreetMember houseMember : houseMembers){
488 				if (houseMember!=null && houseMember.getLocation()!=null){
489 					HouseNumber houseNumber = buildHouseNumberFromAssociatedHouseNumber(houseMember);
490 				OpenStreetMap associatedStreet = openStreetMapDao.getNearestByosmIds(houseMember.getLocation(), streetIds);
491 				if (associatedStreet!=null && houseNumber!=null){
492 					associatedStreet.addHouseNumber(houseNumber);
493 					openStreetMapDao.save(associatedStreet);
494 				} else {
495 					
496 					logger.warn("associated street "+associatedStreet+", or house numer "+houseNumber+" is null");
497 				}
498 				}
499 			}
500 			
501 		}
502 	}
503 
504 	protected HouseNumber processNodeHouseNumber(NodeHouseNumber house) {
505 		if(house==null || house.getLocation()==null){
506 			return null;
507 		}
508 		HouseNumber houseNumber = new HouseNumber();
509 		houseNumber.setNumber(normalizeNumber(house.getHouseNumber()));
510 		houseNumber.setName(house.getName());
511 		houseNumber.setType(HouseNumberType.NODE);
512 		try {
513 			Long id = Long.valueOf(house.getNodeId());
514 			houseNumber.setOpenstreetmapId(id);
515 		} catch (NumberFormatException e) {
516 			//ignore
517 		}
518 		Point location = house.getLocation();
519 		houseNumber.setLocation(location);
520 		OpenStreetMap osm = findNearestStreet(house.getStreetName(),location);
521 		if (osm!=null){
522 					try {
523 						osm.addHouseNumber(houseNumber);
524 						openStreetMapDao.save(osm);
525 					} catch (Exception e) {
526 						logger.error("error processing node housenumber, we ignore it but you should consider it : "+ e.getMessage(),e);
527 					}
528 					return houseNumber;
529 		} else {
530 			logger.warn("can not find node street for name "+house.getStreetName()+", position :"+ location+ " for "+house);
531 		}
532 		return null;
533 	}
534 
535 	protected List<HouseNumber> processInterpolationHouseNumber(InterpolationHouseNumber house) {
536 			//the results
537 			List<HouseNumber> houseNumbers = new ArrayList<HouseNumber>();
538 			boolean multipleInterpolation = false;//boolean to indicate that we 
539 			//interpolate several segment, so we should not add the N2 point
540 			//cause it has already been added by last interpolation
541 			//N1--------N2-------N3
542 		
543 			List<InterpolationMember> membersForSegmentation = new ArrayList<InterpolationMember>();
544 			List<InterpolationMember> members = house.getMembers();
545 			if (members != null) {
546 				for (InterpolationMember member : members) {
547 					if (member.getHouseNumber() != null
548 							&& !"".equals(member.getHouseNumber().trim())) {
549 						// got HN in the member
550 						membersForSegmentation.add(member);
551 						if (membersForSegmentation.size() == 1) {
552 							continue;// we only have one point and need at
553 										// least one other
554 						} else {
555 							int nbInnerPoint = 0;
556 							int increment = 1;
557 							if (house.getInterpolationType() == InterpolationType.alphabetic) {
558 								//WE choose to ignore Alphabetic interpolation due to poor interest
559 							} else {
560 								// odd,even,all=>should be numeric
561 								String firstNumberAsString = membersForSegmentation
562 										.get(0).getHouseNumber();
563 								String lastNumberAsString = membersForSegmentation
564 										.get(membersForSegmentation.size()-1).getHouseNumber();
565 								int firstNumberAsInt = 0;
566 								int lastNumberAsInt = 0;
567 								try {
568 									firstNumberAsInt = Integer
569 											.parseInt(normalizeNumber(firstNumberAsString));
570 									lastNumberAsInt = Integer
571 											.parseInt(normalizeNumber(lastNumberAsString));
572 								} catch (NumberFormatException e) {
573 									logger.warn("interpolation house number "+firstNumberAsString+" and/or "+ lastNumberAsString+"are not numbers");
574 									return houseNumbers;
575 								}
576 								if (house.getInterpolationType() == InterpolationType.even) {// pair
577 									if (firstNumberAsInt % 2 == 1) {
578 										firstNumberAsInt++;
579 										membersForSegmentation.get(0).setHouseNumber(firstNumberAsInt+"");
580 									}
581 									if (lastNumberAsInt % 2 == 1) {
582 										lastNumberAsInt++;
583 										membersForSegmentation.get(membersForSegmentation.size()-1).setHouseNumber(lastNumberAsInt+"");
584 									}
585 									// two even number substracts always give an odd one
586 									nbInnerPoint = Math
587 											.abs((firstNumberAsInt - lastNumberAsInt) / 2) - 1;
588 									if (firstNumberAsInt < lastNumberAsInt){
589 										increment = 2;
590 									} else {
591 										increment = -2;
592 									}
593 
594 								} else if (house.getInterpolationType() == InterpolationType.odd) {// impair
595 									if (firstNumberAsInt % 2 == 0) {
596 										firstNumberAsInt++;
597 										membersForSegmentation.get(0).setHouseNumber(firstNumberAsInt+"");
598 									}
599 									if (lastNumberAsInt % 2 == 0) {
600 										lastNumberAsInt++;
601 										membersForSegmentation.get(membersForSegmentation.size()-1).setHouseNumber(lastNumberAsInt+"");
602 									}
603 									nbInnerPoint = Math
604 											.abs((firstNumberAsInt - lastNumberAsInt) / 2) - 1;
605 									if (firstNumberAsInt < lastNumberAsInt){
606 										increment = 2;
607 									} else {
608 										increment = -2;
609 									}
610 									
611 
612 								} else if (house.getInterpolationType() == InterpolationType.all) {
613 									nbInnerPoint = Math
614 											.abs((firstNumberAsInt - lastNumberAsInt)) - 1;
615 									if (firstNumberAsInt < lastNumberAsInt){
616 										increment = 1;
617 									} else {
618 										increment = -1;
619 									}
620 
621 								}
622 								List<Point> points = new ArrayList<Point>(membersForSegmentation.size());
623 								for (InterpolationMember memberForSegmentation : membersForSegmentation) {
624 									points.add(memberForSegmentation
625 											.getLocation());
626 								}
627 								List<Point> segmentizedPoint = segmentize(
628 										points, nbInnerPoint);
629 								
630 								for (int i =0;i<segmentizedPoint.size();i++){
631 									if (i==0 && multipleInterpolation){
632 										continue;//this point has already been added by previous interpolation
633 									}
634 									HouseNumber houseNumberToAdd = new HouseNumber();
635 									//set the openstretmapid of the first point
636 									if ((i==0 && membersForSegmentation.get(0)!= null && membersForSegmentation.get(0).getId()!=null)){
637 										try {
638 											Long id = Long.valueOf(membersForSegmentation.get(i).getId());
639 											houseNumberToAdd.setOpenstreetmapId(id);
640 										} catch (NumberFormatException e) {
641 											//ignore
642 										}
643 									}
644 									//set the openstretmapid of the last point
645 									if ((i==(segmentizedPoint.size()-1) && membersForSegmentation.get(1)!= null && membersForSegmentation.get(1).getId()!=null)){
646 										try {
647 											Long id = Long.valueOf(membersForSegmentation.get(1).getId());
648 											houseNumberToAdd.setOpenstreetmapId(id);
649 										} catch (NumberFormatException e) {
650 											//ignore
651 										}
652 									}
653 									Point p = segmentizedPoint.get(i);
654 									houseNumberToAdd.setType(HouseNumberType.INTERPOLATION);
655 									houseNumberToAdd.setLocation(p);
656 									houseNumberToAdd.setNumber(firstNumberAsInt+(increment*i)+"");
657 									houseNumbers.add(houseNumberToAdd);
658 								}
659 								//return houseNumbers;
660 								
661 							}
662 							
663 
664 							membersForSegmentation = new ArrayList<InterpolationMember>();
665 							multipleInterpolation=true;
666 							// restart the process with the last point;
667 							membersForSegmentation.add(member);
668 						}
669 					} else {
670 						// no housenumber in the members, it is a point to draw a street
671 						if (membersForSegmentation.size() == 0) {
672 							// we go to the next point to search for a point with HN
673 							continue;
674 						} else {
675 							// we add the member and continue to search for a point with HN;
676 							membersForSegmentation.add(member);
677 						}
678 					}
679 
680 				}
681 			}
682 			return houseNumbers;
683 		}
684 
685 	protected OpenStreetMap findNearestStreet(String streetName, Point location) {
686 		//Openstreetmap has sometimes, for a  same street, several segment, so we do a fulltext search and then search for the nearest based on shape,not nearest point
687 		if (location == null){
688 			return null;
689 		}
690 		if (streetName==null || "".equals(streetName.trim()) || "\"\"".equals(streetName.trim()) ){
691 			return openStreetMapDao.getNearestFrom(location);
692 		}
693 		FulltextQuery query;
694 		try {
695 			query = new FulltextQuery(streetName, Pagination.DEFAULT_PAGINATION, MEDIUM_OUTPUT, 
696 					com.gisgraphy.fulltext.Constants.STREET_PLACETYPE, null);
697 		} catch (IllegalArgumentException e) {
698 			logger.error("can not create a fulltext query for "+streetName+", will return the nearest");
699 			return openStreetMapDao.getNearestFrom(location);
700 		}
701 		query.withAllWordsRequired(false).withoutSpellChecking();
702 		query.around(location);
703 			query.withRadius(SEARCH_DISTANCE);
704 		FulltextResultsDto results;
705 		try {
706 			results = fullTextSearchEngine.executeQuery(query);
707 		} catch (RuntimeException e) {
708 			logger.error("error during fulltext search : "+e.getMessage(),e);
709 			return null;
710 		}
711 		int resultsSize = results.getResultsSize();
712 		List<SolrResponseDto> resultsList = results.getResults();
713 		if (resultsSize == 1) {
714 			SolrResponseDto street = resultsList.get(0);
715 			if (street!=null){
716 				Long openstreetmapId = street.getOpenstreetmap_id();
717 				if (openstreetmapId!=null){
718 					OpenStreetMap osm = openStreetMapDao.getByOpenStreetMapId(openstreetmapId);
719 					if (osm == null) {
720 						logger.warn("can not find street for id "+openstreetmapId);
721 					}
722 					return osm;
723 				}
724 			}
725 		} if (resultsSize > 1) {
726 			return getNearestByIds(resultsList,location);
727 		} else {
728 			return null;
729 		}
730 	}
731 
732 	protected OpenStreetMap getNearestByIds(List<SolrResponseDto> results,Point point) {
733 		List<Long> ids = new ArrayList<Long>();
734 		OpenStreetMap result = null;
735 		if (results!=null){
736 			for (SolrResponseDto dto:results){
737 				if (dto!=null && dto.getOpenstreetmap_id()!=null){
738 					ids.add(dto.getOpenstreetmap_id());
739 				}
740 			}
741 			result = openStreetMapDao.getNearestByosmIds(point, ids);
742 		}
743 		return result;
744 		/*SolrResponseDto candidate=null;
745 		if (results!=null ){
746 			float score =0;
747 			Double smallestDistance = 0D;
748 			for (SolrResponseDto dto: results){
749 				if (score==0){//initialize with first element
750 					score= dto.getScore();
751 					smallestDistance = GeolocHelper.distance(GeolocHelper.createPoint(dto.getLng(), dto.getLat()), point);
752 					candidate=dto;
753 					continue;
754 				}
755 				if ((Math.abs(dto.getScore()-score)*100/score) < SCORE_THRESOLD){//the score is very close => < X%
756 					if (GeolocHelper.distance(GeolocHelper.createPoint(dto.getLng(), dto.getLat()), point) < smallestDistance){//if the dto is closer, we keep it
757 						candidate = dto;
758 					}
759 				} else {//score is too far, and results are sorted by score, we return the candidate
760 					return candidate;
761 				}
762 			}
763 		}
764 		return candidate;
765 		*/
766 	}
767 
768 	protected HouseNumber buildHouseNumberFromAssociatedHouseNumber(
769 			AssociatedStreetMember houseMember) {
770 		HouseNumber houseNumber = new HouseNumber();
771 		if (houseMember.getLocation()!=null){//it is a mandatory field
772 			houseNumber.setLocation(houseMember.getLocation());
773 		} else {
774 			return null;
775 		}
776 		houseNumber.setNumber(normalizeNumber(houseMember.getHouseNumber()));//todo normalize 11 d
777 		Long osmId = null;
778 		try {
779 			osmId = Long.valueOf(houseMember.getId());
780 			houseNumber.setOpenstreetmapId(osmId);
781 		} catch (NumberFormatException e) {
782 			logger.warn(osmId+" is not a valid openstreetmapId");
783 		}
784 		houseNumber.setType(HouseNumberType.ASSOCIATED);
785 		return houseNumber;
786 	}
787 	
788 		
789 	/**
790 	 * @param points a list of point, typically a list of point that represent a street
791 	 * @param nbInnerPoint the number of point to add beetween the startpoint and the endpoint
792 	 * @return the intermediate points that represents segments of same size, if nbInnerPoint=4, 
793 	 * we will get 6 points back
794 	 */
795 	protected List<Point> segmentize(List<Point> points,int nbInnerPoint){
796 		List<Point> result = new ArrayList<Point>();
797 		if (points==null || nbInnerPoint<=0 || points.size()==0){
798 			return result;
799 		}
800 		if (points.size()>=1){
801 			result.add(points.get(0));
802 			
803 		}
804 		if (points.size()==1){
805 			return result;
806 		}
807 		List<Double> distances = new ArrayList<Double>();//5/10/15
808 		double totalDistance = 0;//30
809 		for (int i=0;i<points.size()-1;i++){
810 			double distance = GeolocHelper.distance(points.get(i), points.get(i+1));
811 			distances.add(distance);
812 			totalDistance+=distance;
813 		}
814 		int nbSegment = nbInnerPoint+1;//4+1
815 		int nbpoints = nbInnerPoint+1;//4+1
816 		double segmentLength = totalDistance/nbSegment;//30/5=6
817 		int currentSubSegment =1;
818 		double currentSubLength=0;
819 		for (int i=1;i<=nbpoints;i++){
820 			while(currentSubLength+distances.get(currentSubSegment-1) <i*segmentLength){
821 				currentSubLength+=distances.get(currentSubSegment-1);
822 				currentSubSegment++;
823 				if (currentSubSegment>distances.size()){
824 					return result;
825 				}
826 			}
827 			double distanceLeft = (i*segmentLength)-currentSubLength;
828 			double ratio = distanceLeft/(distances.get(currentSubSegment-1));
829 			if (ratio <= 0){
830 				//it is the last point
831 				result.add(points.get(points.size()-1));
832 			} else {
833 				//we put it at the correct ratio
834 				Point p1=points.get(currentSubSegment-1);
835 				Point p2=points.get(currentSubSegment);
836 				Double lng = (p2.getX()-p1.getX())*ratio+p1.getX();
837 				Double lat = (p2.getY()-p1.getY())*ratio+p1.getY();
838 				Point p = GeolocHelper.createPoint(lng.floatValue(), lat.floatValue());
839 				result.add(p);
840 			}
841 		}
842 		return result;
843 	}
844 
845 
846 	/*
847 	 * (non-Javadoc)
848 	 * 
849 	 * @see
850 	 * com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#shouldBeSkiped
851 	 * ()
852 	 */
853 	@Override
854 	public boolean shouldBeSkipped() {
855 		return !importerConfig.isOpenstreetmapHouseNumberImporterEnabled();
856 	}
857 
858 	/*
859 	 * (non-Javadoc)
860 	 * 
861 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#
862 	 * setCommitFlushMode()
863 	 */
864 	@Override
865 	protected void setCommitFlushMode() {
866 		this.openStreetMapDao.setFlushMode(FlushMode.COMMIT);
867 		this.houseNumberDao.setFlushMode(FlushMode.COMMIT);
868 	}
869 
870 	/*
871 	 * (non-Javadoc)
872 	 * 
873 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#
874 	 * shouldIgnoreComments()
875 	 */
876 	@Override
877 	protected boolean shouldIgnoreComments() {
878 		return false;
879 	}
880 
881 	/*
882 	 * (non-Javadoc)
883 	 * 
884 	 * @see com.gisgraphy.domain.geoloc.importer.AbstractImporterProcessor#
885 	 * shouldIgnoreFirstLine()
886 	 */
887 	@Override
888 	protected boolean shouldIgnoreFirstLine() {
889 		return false;
890 	}
891 
892 	/*
893 	 * (non-Javadoc)
894 	 * 
895 	 * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#rollback()
896 	 */
897 	public List<NameValueDTO<Integer>> rollback() {
898 		List<NameValueDTO<Integer>> deletedObjectInfo = new ArrayList<NameValueDTO<Integer>>();
899 		logger.info("reseting  house number importer...");
900 		int deleted = houseNumberDao.deleteAll();
901 		if (deleted != 0) {
902 			deletedObjectInfo.add(new NameValueDTO<Integer>(houseNumberDao.getPersistenceClass().getSimpleName(), deleted));
903 		}
904 		logger.info(deleted + " house number entities have been deleted");
905 		resetStatus();
906 		return deletedObjectInfo;
907 	}
908 
909 	
910 
911 	@Override
912 	// TODO test
913 	protected void tearDown() {
914 		super.tearDown();
915 		FullTextSearchEngine.disableLogging=false;
916 		this.statusMessage = internationalisationService.getString("import.fulltext.optimize");
917 		solRSynchroniser.optimize();
918 	}
919 
920 
921 	@Required
922 	public void setHouseNumberDao(IhouseNumberDao houseNumberDao) {
923 		this.houseNumberDao = houseNumberDao;
924 	}
925 	
926 	@Required
927 	public void setOpenStreetMapDao(IOpenStreetMapDao openStreetMapDao) {
928 		this.openStreetMapDao = openStreetMapDao;
929 	}
930 	
931 	@Required
932 	public void setFullTextSearchEngine(IFullTextSearchEngine fullTextSearchEngine) {
933 		this.fullTextSearchEngine = fullTextSearchEngine;
934 	}
935 
936 	@Required
937 	public void setSolRSynchroniser(ISolRSynchroniser solRSynchroniser) {
938 		this.solRSynchroniser = solRSynchroniser;
939 	}
940 
941 
942 }