View Javadoc
1   /*******************************************************************************
2    * Gisgraphy Project 
3    *  
4    *    This library is free software; you can redistribute it and/or
5    *    modify it under the terms of the GNU Lesser General Public
6    *    License as published by the Free Software Foundation; either
7    *    version 2.1 of the License, or (at your option) any later version.
8    *  
9    *    This library is distributed in the hope that it will be useful,
10   *    but WITHOUT ANY WARRANTY; without even the implied warranty of
11   *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12   *    Lesser General Public License for more details.
13   *  
14   *    You should have received a copy of the GNU Lesser General Public
15   *    License along with this library; if not, write to the Free Software
16   *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
17   *  
18   *   Copyright 2008  Gisgraphy project 
19   * 
20   *   David Masclet <davidmasclet@gisgraphy.com>
21   ******************************************************************************/
22  package com.gisgraphy.geocoding;
23  
24  import static com.gisgraphy.fulltext.Constants.ADDRESSES_PLACETYPE;
25  import static com.gisgraphy.helper.StringHelper.isEmptyString;
26  import static com.gisgraphy.helper.StringHelper.isNotEmptyString;
27  
28  import java.io.ByteArrayOutputStream;
29  import java.io.OutputStream;
30  import java.io.UnsupportedEncodingException;
31  import java.util.ArrayList;
32  import java.util.Collections;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.regex.Matcher;
37  import java.util.regex.Pattern;
38  
39  import org.slf4j.Logger;
40  import org.slf4j.LoggerFactory;
41  import org.springframework.beans.factory.annotation.Autowired;
42  import org.springframework.stereotype.Service;
43  
44  import com.gisgraphy.addressparser.Address;
45  import com.gisgraphy.addressparser.AddressQuery;
46  import com.gisgraphy.addressparser.AddressResultsDto;
47  import com.gisgraphy.addressparser.IAddressParserService;
48  import com.gisgraphy.addressparser.StructuredAddressQuery;
49  import com.gisgraphy.addressparser.exception.AddressParserException;
50  import com.gisgraphy.domain.geoloc.entity.Adm;
51  import com.gisgraphy.domain.geoloc.entity.City;
52  import com.gisgraphy.domain.geoloc.entity.CitySubdivision;
53  import com.gisgraphy.domain.geoloc.entity.Street;
54  import com.gisgraphy.domain.valueobject.Constants;
55  import com.gisgraphy.domain.valueobject.GisgraphyConfig;
56  import com.gisgraphy.domain.valueobject.Output;
57  import com.gisgraphy.domain.valueobject.Output.OutputStyle;
58  import com.gisgraphy.domain.valueobject.Pagination;
59  import com.gisgraphy.fulltext.FulltextQuery;
60  import com.gisgraphy.fulltext.FulltextResultsDto;
61  import com.gisgraphy.fulltext.IFullTextSearchEngine;
62  import com.gisgraphy.fulltext.SolrResponseDto;
63  import com.gisgraphy.fulltext.SolrResponseDtoDistanceComparator;
64  import com.gisgraphy.helper.GeolocHelper;
65  import com.gisgraphy.helper.StringHelper;
66  import com.gisgraphy.importer.ImporterConfig;
67  import com.gisgraphy.serializer.UniversalSerializer;
68  import com.gisgraphy.serializer.common.UniversalSerializerConstant;
69  import com.gisgraphy.service.IStatsUsageService;
70  import com.gisgraphy.stats.StatsUsageType;
71  import com.gisgraphy.street.HouseNumberDto;
72  import com.gisgraphy.street.HouseNumberUtil;
73  import com.vividsolutions.jts.geom.Point;
74  
75  /**
76   * 
77   * Geocode internationnal address via gisgraphy services
78   * 
79   * @author <a href="mailto:david.masclet@gisgraphy.com">David Masclet</a>
80   * 
81   */
82  @Service
83  public class GeocodingService implements IGeocodingService {
84  	private IStatsUsageService statsUsageService;
85  	private ImporterConfig importerConfig;
86  	private IAddressParserService addressParser;
87  	private IFullTextSearchEngine fullTextSearchEngine;
88  	private GisgraphyConfig gisgraphyConfig;
89  
90  	public final static int ACCEPT_DISTANCE_BETWEEN_CITY_AND_STREET = 30000;
91  	public final static Output LONG_OUTPUT = Output.withDefaultFormat().withStyle(OutputStyle.LONG);
92  	public final static Output MEDIUM_OUTPUT = Output.withDefaultFormat().withStyle(OutputStyle.MEDIUM);
93  	public final static Pagination ONE_RESULT_PAGINATION = Pagination.paginate().from(0).to(1);
94  	public final static Pagination FIVE_RESULT_PAGINATION = Pagination.paginate().from(0).to(5);
95  	public final static SolrResponseDtoDistanceComparator comparator = new SolrResponseDtoDistanceComparator();
96  	public final static Pattern HOUSENUMBERPATTERN = Pattern.compile("((?:^\\b\\d{1,4})(?!(?:st\\b|th\\b|rd\\b|nd\\b))|(((?:\\b\\d{1,4}))\\b(?:[\\s,;]+)(?!(?:st\\b|th\\b|rd\\b|nd\\b))(?=\\w+)+?)|\\s?(?:\\b\\d{1,4}$))");
97  
98  	/**
99  	 * The logger
100 	 */
101 	protected static final Logger logger = LoggerFactory.getLogger(GeocodingService.class);
102 
103 	/*
104 	 * (non-Javadoc)
105 	 * 
106 	 * @see
107 	 * com.gisgraphy.geocoding.IGeocodingService#geocodeAndSerialize(com.gisgraphy
108 	 * .addressparser.AddressQuery, java.io.OutputStream)
109 	 */
110 	public void geocodeAndSerialize(AddressQuery query, OutputStream outputStream) throws GeocodingException {
111 		if (query == null) {
112 			throw new GeocodingException("Can not geocode a null query");
113 		}
114 		if (outputStream == null) {
115 			throw new GeocodingException("Can not serialize into a null outputStream");
116 		}
117 		AddressResultsDto geolocResultsDto = geocode(query);
118 		Map<String, Object> extraParameter = new HashMap<String, Object>();
119 		// extraParameter.put(GeolocResultsDtoSerializer.START_PAGINATION_INDEX_EXTRA_PARAMETER,
120 		// query.getFirstPaginationIndex());
121 		extraParameter.put(UniversalSerializerConstant.CALLBACK_METHOD_NAME, query.getCallback());
122 		UniversalSerializer.getInstance().write(outputStream, geolocResultsDto, false, extraParameter, query.getFormat());
123 	}
124 
125 	/*
126 	 * (non-Javadoc)
127 	 * 
128 	 * @see
129 	 * com.gisgraphy.geocoding.IGeocodingService#geocodeToString(com.gisgraphy
130 	 * .addressparser.AddressQuery)
131 	 */
132 	public String geocodeToString(AddressQuery query) throws GeocodingException {
133 		if (query == null) {
134 			throw new GeocodingException("Can not geocode a null query");
135 		}
136 		ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
137 		geocodeAndSerialize(query, outputStream);
138 		try {
139 			return outputStream.toString(Constants.CHARSET);
140 		} catch (UnsupportedEncodingException e) {
141 			throw new RuntimeException("unknow encoding " + Constants.CHARSET);
142 		}
143 	}
144 
145 	/*
146 	 * (non-Javadoc)
147 	 * 
148 	 * @see com.gisgraphy.geocoding.IGeocodingService#geocode(java.lang.String)
149 	 */
150 	public AddressResultsDto geocode(AddressQuery query) throws GeocodingException {
151 		if (query == null) {
152 			throw new GeocodingException("Can not geocode a null query");
153 		}
154 		logger.info(query.toString());
155 		String countryCode = query.getCountry();
156 		if (countryCode !=null  && countryCode.trim().length() != 2) {
157 			throw new GeocodingException("countrycode should have two letters : " + countryCode);
158 		}
159 		if (query instanceof StructuredAddressQuery){
160 			Address address = ((StructuredAddressQuery)query).getStructuredAddress();
161 			if (logger.isDebugEnabled()) {
162 				logger.debug("structured address to geocode : '" + address + "' for country code : " + countryCode);
163 			}
164 			return geocode(address, countryCode);
165 		}
166 		String rawAddress = query.getAddress();
167 		if (isEmptyString(rawAddress)) {
168 			throw new GeocodingException("Can not geocode a null or empty address");
169 		}
170 		if (logger.isDebugEnabled()) {
171 			logger.debug("Raw address to geocode : '" + rawAddress + "' for country code : " + countryCode);
172 		}
173 		Long startTime = System.currentTimeMillis();
174 		AddressQuery addressQuery = new AddressQuery(rawAddress, countryCode);
175 		AddressResultsDto addressResultDto = null;
176 		logger.debug("is postal address : " +query.isPostal());
177 		if (gisgraphyConfig.useAddressParserWhenGeocoding || query.isPostal()) {
178 			try {
179 				logger.debug("address parser is enabled");
180 				addressResultDto = addressParser.execute(addressQuery);
181 			} catch (AddressParserException e) {
182 				logger.error("An error occurs during parsing of address" + e.getMessage(), e);
183 			}
184 		}
185 		if (addressResultDto != null && addressResultDto.getResult().size() >= 1 && isGeocodable(addressResultDto.getResult().get(0))) {
186 			if (logger.isDebugEnabled()) {
187 				logger.debug("successfully parsed address : " + rawAddress + " : " + addressResultDto.getResult().get(0));
188 			}
189 			Address address = addressResultDto.getResult().get(0);
190 			AddressResultsDto addressesDto = geocode(address, countryCode);
191 			addressesDto.setParsedAddress(address);
192 			return addressesDto;
193 		} else if (importerConfig.isOpenStreetMapFillIsIn()) {
194 			logger.debug("is_in is active");
195 			statsUsageService.increaseUsage(StatsUsageType.GEOCODING);
196 			AddressResultsDto results;
197 			List<SolrResponseDto> aproximativeMatches = findStreetInText(rawAddress, countryCode, null);
198 			String houseNumber = findHouseNumber(rawAddress, countryCode);
199 			if (GisgraphyConfig.searchForExactMatchWhenGeocoding) {
200 				List<SolrResponseDto> exactMatches = findExactMatches(rawAddress, countryCode);
201 				List<SolrResponseDto> mergedResults = mergeSolrResponseDto(exactMatches, aproximativeMatches);
202 				results = buildAddressResultDtoFromSolrResponseDto(mergedResults, houseNumber);
203 			} else {
204 				results = buildAddressResultDtoFromStreetsAndCities(aproximativeMatches, null, houseNumber);
205 			}
206 			Long endTime = System.currentTimeMillis();
207 			long qTime = endTime - startTime;
208 			results.setQTime(qTime);
209 			logger.info("geocoding of "+query + " and country="+countryCode+" took " + (qTime) + " ms and returns "
210 					+ results.getNumFound() + " results");
211 			return results;
212 		} else {
213 			logger.debug("is_in is inactive");
214 			// we call the stats here because we don't want to call it twice
215 			// when we call geocode before
216 			statsUsageService.increaseUsage(StatsUsageType.GEOCODING);
217 			if (logger.isDebugEnabled()) {
218 				logger.debug("unsuccessfull address parsing of  : '" + rawAddress + "'");
219 			}
220 			List<SolrResponseDto> cities = null;
221 			SolrResponseDto city = null;
222 			cities = findCitiesInText(rawAddress, countryCode);
223 			if (cities != null && cities.size() > 0) {
224 				city = cities.get(0);
225 			}
226 			if (city == null) {
227 				if (logger.isDebugEnabled()) {
228 					logger.debug(" no city found for '" + rawAddress + "'");
229 				}
230 				List<SolrResponseDto> streets = findStreetInText(rawAddress, countryCode, null);
231 				String houseNumber = findHouseNumber(rawAddress, countryCode);
232 				AddressResultsDto results = buildAddressResultDtoFromStreetsAndCities(streets, cities, houseNumber);
233 				Long endTime = System.currentTimeMillis();
234 				long qTime = endTime - startTime;
235 				results.setQTime(qTime);
236 				logger.info("geocoding of "+query + " and country="+countryCode+" took " + (qTime) + " ms and returns "
237 						+ results.getNumFound() + " results");
238 				return results;
239 
240 			} else {
241 				if (logger.isDebugEnabled()) {
242 					logger.debug("found city : " + city.getName() + " for '" + rawAddress + "'");
243 				}
244 				Point cityLocation = GeolocHelper.createPoint(city.getLng().floatValue(), city.getLat().floatValue());
245 				List<SolrResponseDto> streets = null;
246 				if (importerConfig.isOpenStreetMapFillIsIn()) {
247 					streets = findStreetInText(rawAddress, countryCode, cityLocation);
248 				} else {
249 					String addressNormalized = StringHelper.normalize(rawAddress);
250 					String cityNormalized = StringHelper.normalize(city.getName());
251 					String addressNormalizedWithoutCity = addressNormalized.replace(cityNormalized, "");
252 					if (isNotEmptyString(addressNormalizedWithoutCity)) {
253 						if (logger.isDebugEnabled()) {
254 							logger.debug("normalized address without city : " + addressNormalizedWithoutCity);
255 						}
256 						streets = findStreetInText(addressNormalizedWithoutCity, countryCode, cityLocation);
257 					}
258 				}
259 				String houseNumber = findHouseNumber(rawAddress, countryCode);
260 				AddressResultsDto results = buildAddressResultDtoFromStreetsAndCities(streets, cities, houseNumber);
261 				Long endTime = System.currentTimeMillis();
262 				long qTime = endTime - startTime;
263 				results.setQTime(qTime);
264 				logger.info("geocoding of "+query + " and country="+countryCode+" took " + (qTime) + " ms and returns "
265 						+ results.getNumFound() + " results");
266 				return results;
267 
268 			}
269 		}
270 
271 	}
272 
273 	protected boolean isGeocodable(Address address) {
274 		if (isEmptyString(address.getStreetName()) && isEmptyString(address.getState()) && isEmptyString(address.getCity()) && isEmptyString(address.getZipCode()) && isEmptyString(address.getPostTown())) {
275 			logger.info(address+" is no geocodable");
276 			return false;
277 		}
278 		return true;
279 	}
280 
281 	/*
282 	 * (non-Javadoc)
283 	 * 
284 	 * @see
285 	 * com.gisgraphy.geocoding.IGeocodingService#geocode(com.gisgraphy.addressparser
286 	 * .Address)
287 	 */
288 	public AddressResultsDto geocode(Address address, String countryCode) throws GeocodingException {
289 		if (address == null) {
290 			throw new GeocodingException("Can not geocode a null address");
291 		}
292 		if (countryCode!=null &&  countryCode.trim().length() != 2) {
293 			throw new GeocodingException("wrong countrycode : " + countryCode);
294 		}
295 		if (isIntersection(address)) {
296 			throw new GeocodingException("street intersection is not managed yet");
297 		}
298 		if (!isGeocodable(address)) {
299 			throw new GeocodingException("City, street name, posttown and zip is not set, we got too less informations to geocode ");
300 		}
301 		statsUsageService.increaseUsage(StatsUsageType.GEOCODING);
302 		Long startTime = System.currentTimeMillis();
303 		if (isEmptyString(address.getCity()) && isEmptyString(address.getZipCode()) && isEmptyString(address.getPostTown())) {
304 			List<SolrResponseDto> streets = findStreetInText(address.getStreetName(), countryCode, null);
305 			AddressResultsDto results = buildAddressResultDtoFromStreetsAndCities(streets, null, address.getHouseNumber());
306 			//results.setParsedAddress(address);
307 			Long endTime = System.currentTimeMillis();
308 			long qTime = endTime - startTime;
309 			results.setQTime(qTime);
310 			logger.info("geocoding of "+address + " and country="+countryCode+" took " + (qTime) + " ms and returns "
311 					+ results.getNumFound() + " results");
312 			return results;
313 		} else {
314 			String bestSentence = getBestCitySearchSentence(address);
315 			List<SolrResponseDto> cities = null;
316 			cities = findCitiesInText(bestSentence, countryCode);
317 			Point cityLocation = null;
318 			if (cities != null && cities.size() > 0 && cities.get(0) != null) {
319 				logger.debug("city found "+cities.get(0).getName()+"/"+cities.get(0).getFeature_id());
320 				cityLocation = GeolocHelper.createPoint(cities.get(0).getLng().floatValue(), cities.get(0).getLat().floatValue());
321 			}
322 			// TODO iterate over cities
323 			List<SolrResponseDto> fulltextResultsDto = null;
324 			if (address.getStreetName() != null) {
325 				String streetSentenceToSearch = address.getStreetName();
326 				if (address.getPreDirection()!= null) {
327 					streetSentenceToSearch += address.getPreDirection() +" "+ address.getCity();
328 				}
329 				if (address.getPostDirection()!= null) {
330 					streetSentenceToSearch += streetSentenceToSearch+" " +address.getPostDirection();
331 				}
332 				fulltextResultsDto = findStreetInText(streetSentenceToSearch, countryCode, cityLocation);
333 			}
334 			AddressResultsDto results = buildAddressResultDtoFromStreetsAndCities(fulltextResultsDto, cities, address.getHouseNumber());
335 			//results.setParsedAddress(address);
336 			Long endTime = System.currentTimeMillis();
337 			long qTime = endTime - startTime;
338 			results.setQTime(qTime);
339 			logger.info("geocoding of "+address + " and country="+countryCode+" took " + (qTime) + " ms and returns "
340 					+ results.getNumFound() + " results");
341 			return results;
342 		}
343 	}
344 
345 	protected String getBestCitySearchSentence(Address address) {
346 		String sentence = "";
347 		if (isNotEmptyString(address.getCity())) {
348 			sentence += " " + address.getCity();
349 		} else if (isNotEmptyString(address.getPostTown())){
350 			sentence += " " + address.getPostTown();
351 		}
352 		if (isNotEmptyString(address.getZipCode())) {
353 			sentence += " " + address.getZipCode();
354 		}
355 		String dependentLocality = address.getDependentLocality();
356 		String state = address.getState();
357 		String choice = "";
358 		if (isEmptyString(state) && isNotEmptyString(dependentLocality)) {
359 			choice = " " + dependentLocality;
360 		} else if (isNotEmptyString(state) && isEmptyString(dependentLocality)) {
361 			choice = " " + state;
362 		} else if (isNotEmptyString(state) && isNotEmptyString(dependentLocality)) {
363 			choice = " " + state + " " + dependentLocality;
364 		}
365 		return new String(sentence + choice).trim();
366 	}
367 
368 	
369 
370 	protected AddressResultsDto buildAddressResultDtoFromStreetsAndCities(List<SolrResponseDto> streets, List<SolrResponseDto> cities, String houseNumberToFind) {
371 		List<Address> addresses = new ArrayList<Address>();
372 
373 		if (streets != null && streets.size() > 0) {
374 			if (logger.isDebugEnabled()) {
375 				logger.debug("found " + streets.size() + " streets");
376 			}
377 			SolrResponseDto city = null;
378 			if (cities != null && cities.size() > 0) {
379 				city = cities.get(0);
380 			}
381 			String lastName=null;
382 			boolean housenumberFound =false;
383 			int numberOfStreetThatHaveTheSameName = 0;
384 			for (SolrResponseDto street : streets) {
385 				Address address = new Address();
386 
387 				address.setLat(street.getLat());
388 				address.setLng(street.getLng());
389 				address.setId(street.getFeature_id());
390 				address.setCountryCode(street.getCountry_code());
391 
392 				String streetName = street.getName();
393 				if (logger.isDebugEnabled() && streets != null) {
394 					logger.debug("=>street : " + streetName +" ("+street.getScore()+")");
395 				}
396 				address.setStreetName(streetName);
397 				address.setStreetType(street.getStreet_type());
398 				String is_in = street.getIs_in();
399 				if (!isEmptyString(is_in)) {
400 					address.setCity(is_in);
401 					address.setState(street.getIs_in_adm());
402 					if (street.getIs_in_zip()!=null && street.getIs_in_zip().size()>=1){
403 						address.setZipCode(street.getIs_in_zip().iterator().next());
404 					}
405 					address.setDependentLocality(street.getIs_in_place());
406 				} else {
407 					populateAddressFromCity(city, address);
408 				}
409 				//now search for house number!
410 				List<HouseNumberDto> houseNumbersList = street.getHouse_numbers();
411 				if(houseNumberToFind!=null && houseNumbersList!=null && houseNumbersList.size()>0){
412 				if (!isEmptyString(streetName)){ 
413 					if(streetName.equals(lastName) && city!=null){//probably the same street
414 						if (housenumberFound){
415 							continue;
416 							//do nothing it has already been found in the street
417 							//TODO do we have to search and if we find, we add it?
418 						}else {
419 							numberOfStreetThatHaveTheSameName++;
420 							HouseNumberDto houseNumber = searchHouseNumber(houseNumberToFind,houseNumbersList,street.getCountry_code());
421 							if (houseNumber !=null){
422 								housenumberFound=true;
423 								address.setHouseNumber(houseNumber.getNumber());
424 								address.setLat(houseNumber.getLocation().getY());
425 								address.setLng(houseNumber.getLocation().getX());
426 								//remove the last results added
427 								for (numberOfStreetThatHaveTheSameName--;numberOfStreetThatHaveTheSameName>=0;numberOfStreetThatHaveTheSameName--){
428 									addresses.remove(addresses.size()-1-numberOfStreetThatHaveTheSameName);
429 								}
430 							} else {
431 								housenumberFound=false;
432 							}
433 						}
434 					} else { //the streetName is different, 
435 						numberOfStreetThatHaveTheSameName=0;
436 						HouseNumberDto houseNumber = searchHouseNumber(houseNumberToFind,houseNumbersList,street.getCountry_code());
437 						if (houseNumber !=null){
438 							housenumberFound=true;
439 							address.setHouseNumber(houseNumber.getNumber());
440 							address.setLat(houseNumber.getLocation().getY());
441 							address.setLng(houseNumber.getLocation().getX());
442 						} else {
443 							housenumberFound=false;
444 						}
445 					}
446 				} else {//streetname is null, we search for housenumber anyway
447 					numberOfStreetThatHaveTheSameName=0;
448 					HouseNumberDto houseNumber = searchHouseNumber(houseNumberToFind,houseNumbersList,street.getCountry_code());
449 					if (houseNumber !=null){
450 						housenumberFound=true;
451 						address.setHouseNumber(houseNumber.getNumber());
452 						address.setLat(houseNumber.getLocation().getY());
453 						address.setLng(houseNumber.getLocation().getX());
454 					} else {
455 						housenumberFound=false;
456 					}
457 				}
458 				}
459 				lastName=streetName;
460 				address.getGeocodingLevel();//force calculation of geocodingLevel
461 				addresses.add(address);
462 
463 			}
464 		} else {
465 			if (cities != null && cities.size() > 0) {
466 				if (logger.isDebugEnabled()) {
467 					logger.debug("No street found, only cities");
468 				}
469 				for (SolrResponseDto city : cities) {
470 					// the best we can do is city
471 					Address address = buildAddressFromCity(city);
472 					address.getGeocodingLevel();//force calculation of geocodingLevel
473 					addresses.add(address);
474 				}
475 			} else if (logger.isDebugEnabled()) {
476 				logger.debug("No street and no city found");
477 			}
478 		}
479 		return new AddressResultsDto(addresses, 0L);
480 	}
481 
482 	protected HouseNumberDto searchHouseNumber(String houseNumberToFind, List<HouseNumberDto> houseNumbersList,String countryCode) {
483 		if(houseNumberToFind==null || houseNumbersList==null || houseNumbersList.size()==0){
484 			return null;
485 		}
486 		String HouseNumberToFind;
487 		if (countryCode!=null && ("SK".equalsIgnoreCase(countryCode) || "CZ".equalsIgnoreCase(countryCode))){
488 			HouseNumberToFind = HouseNumberUtil.normalizeSkCzNumber(houseNumberToFind);
489 		} else {
490 			HouseNumberToFind = HouseNumberUtil.normalizeNumber(houseNumberToFind);
491 		}
492 		for (HouseNumberDto candidate :houseNumbersList){
493 			if (candidate!=null && candidate.getNumber()!=null && candidate.getNumber().equals(HouseNumberToFind)){
494 				logger.info("house number candidate found : "+candidate.getNumber());
495 				return candidate;
496 			}
497 		}
498 		logger.info("no house number candidate found for "+houseNumberToFind);
499 		return null;
500 	}
501 
502 	protected AddressResultsDto buildAddressResultDtoFromSolrResponseDto(List<SolrResponseDto> solResponseDtos, String houseNumberToFind) {
503 		List<Address> addresses = new ArrayList<Address>();
504 
505 		if (solResponseDtos != null && solResponseDtos.size() > 0) {
506 			if (logger.isDebugEnabled()) {
507 				logger.debug("found " + solResponseDtos.size() + " results");
508 			}
509 			String lastName=null;
510 			String lastIsin=null;
511 			boolean housenumberFound =false;
512 			int numberOfStreetThatHaveTheSameName = 0;
513 			for (SolrResponseDto solrResponseDto : solResponseDtos) {
514 				Address address = new Address();
515 				if (solrResponseDto == null) {
516 					continue;
517 				}
518 				address.setName(solrResponseDto.getName());
519 				address.setLat(solrResponseDto.getLat());
520 				address.setLng(solrResponseDto.getLng());
521 				address.setId(solrResponseDto.getFeature_id());
522 				address.setCountryCode(solrResponseDto.getCountry_code());
523 				if (solrResponseDto.getPlacetype().equalsIgnoreCase(Adm.class.getSimpleName())) {
524 					address.setState(solrResponseDto.getName());
525 				}else if (solrResponseDto.getAdm2_name() != null) {
526 					address.setState(solrResponseDto.getAdm2_name());
527 				} else if (solrResponseDto.getAdm1_name() != null) {
528 					address.setState(solrResponseDto.getAdm1_name());
529 				} 
530 				if (solrResponseDto.getZipcodes() != null && solrResponseDto.getZipcodes().size() > 0) {
531 					address.setZipCode(solrResponseDto.getZipcodes().iterator().next());
532 				}
533 				if (solrResponseDto.getPlacetype().equalsIgnoreCase(Street.class.getSimpleName())) {
534 					String streetName = solrResponseDto.getName();
535 					String isIn = solrResponseDto.getIs_in();
536 					if (!isEmptyString(solrResponseDto.getName())){ 
537 						if(streetName.equals(lastName) && isIn!=null && isIn.equalsIgnoreCase(lastIsin)){//probably the same street
538 							if (housenumberFound){
539 								continue;
540 								//do nothing it has already been found in the street
541 								//TODO do we have to search and if we find, we add it?
542 							}else {
543 								numberOfStreetThatHaveTheSameName++;
544 							address.setStreetName(solrResponseDto.getName());
545 							address.setStreetType(solrResponseDto.getStreet_type());
546 							address.setCity(solrResponseDto.getIs_in());
547 							address.setState(solrResponseDto.getIs_in_adm());
548 							if (solrResponseDto.getIs_in_zip()!=null && solrResponseDto.getIs_in_zip().size()>=1){
549 								address.setZipCode(solrResponseDto.getIs_in_zip().iterator().next());
550 							}
551 							address.setDependentLocality(solrResponseDto.getIs_in_place());
552 							//now search for houseNumber
553 							List<HouseNumberDto> houseNumbersList = solrResponseDto.getHouse_numbers();
554 							if(houseNumberToFind!=null && houseNumbersList!=null && houseNumbersList.size()>0){
555 								HouseNumberDto houseNumber = searchHouseNumber(houseNumberToFind,houseNumbersList,solrResponseDto.getCountry_code());
556 								if (houseNumber !=null){
557 									housenumberFound=true;
558 									address.setHouseNumber(houseNumber.getNumber());
559 									address.setLat(houseNumber.getLocation().getY());
560 									address.setLng(houseNumber.getLocation().getX());
561 									//remove the last results added
562 									for (numberOfStreetThatHaveTheSameName--;numberOfStreetThatHaveTheSameName>=0;numberOfStreetThatHaveTheSameName--){
563 										addresses.remove(addresses.size()-1-numberOfStreetThatHaveTheSameName);
564 									}
565 								} else{
566 									housenumberFound=false;
567 								}
568 							}
569 						}
570 						} else { //the streetName is different, 
571 							
572 							//remove the last results added
573 							for (numberOfStreetThatHaveTheSameName--;numberOfStreetThatHaveTheSameName>=0;numberOfStreetThatHaveTheSameName--){
574 								addresses.remove(addresses.size()-1-numberOfStreetThatHaveTheSameName);
575 							}
576 							numberOfStreetThatHaveTheSameName=0;
577 							//populate fields
578 							address.setStreetName(solrResponseDto.getName());
579 							address.setStreetType(solrResponseDto.getStreet_type());
580 							address.setCity(solrResponseDto.getIs_in());
581 							address.setState(solrResponseDto.getIs_in_adm());
582 							if (solrResponseDto.getIs_in_zip()!=null && solrResponseDto.getIs_in_zip().size()>=1){
583 								address.setZipCode(solrResponseDto.getIs_in_zip().iterator().next());
584 							}
585 							address.setDependentLocality(solrResponseDto.getIs_in_place());
586 							//search for housenumber
587 							List<HouseNumberDto> houseNumbersList = solrResponseDto.getHouse_numbers();
588 							if(houseNumberToFind!=null && houseNumbersList!=null && houseNumbersList.size()>0){
589 							HouseNumberDto houseNumber = searchHouseNumber(houseNumberToFind,houseNumbersList,solrResponseDto.getCountry_code());
590 							if (houseNumber !=null){
591 								housenumberFound=true;
592 								address.setHouseNumber(houseNumber.getNumber());
593 								address.setLat(houseNumber.getLocation().getY());
594 								address.setLng(houseNumber.getLocation().getX());
595 							} else {
596 								housenumberFound=false;
597 							}
598 							}
599 						}
600 			  } else {//streetname is null, we search for housenumber anyway
601 				  address.setStreetType(solrResponseDto.getStreet_type());
602 					address.setCity(solrResponseDto.getIs_in());
603 					address.setState(solrResponseDto.getIs_in_adm());
604 					if (solrResponseDto.getIs_in_zip()!=null && solrResponseDto.getIs_in_zip().size()>=1){
605 						address.setZipCode(solrResponseDto.getIs_in_zip().iterator().next());
606 					}
607 					address.setDependentLocality(solrResponseDto.getIs_in_place());
608 				  List<HouseNumberDto> houseNumbersList = solrResponseDto.getHouse_numbers();
609 					if(houseNumberToFind!=null && houseNumbersList!=null && houseNumbersList.size()>0){
610 					HouseNumberDto houseNumber = searchHouseNumber(houseNumberToFind,houseNumbersList,solrResponseDto.getCountry_code());
611 					if (houseNumber !=null){
612 						housenumberFound=true;
613 						address.setHouseNumber(houseNumber.getNumber());
614 						address.setLat(houseNumber.getLocation().getY());
615 						address.setLng(houseNumber.getLocation().getX());
616 					} else {
617 						housenumberFound=false;
618 					}
619 				}
620 			  }
621 					lastName=streetName;
622 					lastIsin = isIn;
623 				} else if (solrResponseDto.getPlacetype().equalsIgnoreCase(City.class.getSimpleName())){
624 					address.setCity(solrResponseDto.getName());
625 				} else if (solrResponseDto.getPlacetype().equalsIgnoreCase(CitySubdivision.class.getSimpleName())) {
626 					address.setQuarter(solrResponseDto.getName());
627 				}
628 				 
629 				if (logger.isDebugEnabled() && solrResponseDto != null) {
630 					logger.debug("=>place (" + solrResponseDto.getFeature_id()+") : "+solrResponseDto.getName() +" in "+solrResponseDto.getIs_in());
631 				}
632 				address.getGeocodingLevel();//force calculation of geocodingLevel
633 				addresses.add(address);
634 
635 			}
636 		}
637 		return new AddressResultsDto(addresses, 0L);
638 	}
639 
640 	private Address buildAddressFromCity(SolrResponseDto city) {
641 		Address address = new Address();
642 		address.setLat(city.getLat());
643 		address.setLng(city.getLng());
644 		populateAddressFromCity(city, address);
645 		address.setId(city.getFeature_id());
646 		return address;
647 	}
648 	
649 
650 	protected void populateAddressFromCity(SolrResponseDto city, Address address) {
651 		if (city != null) {
652 			address.setCity(city.getName());
653 			if (city.getAdm2_name() != null) {
654 				address.setState(city.getAdm2_name());
655 			} else if (city.getAdm1_name() != null) {
656 				address.setState(city.getAdm1_name());
657 			} else if (city.getIs_in_adm()!=null){
658 				address.setState(city.getIs_in_adm());
659 			}
660 			if (city.getZipcodes() != null && city.getZipcodes().size() > 0) {
661 				address.setZipCode(city.getZipcodes().iterator().next());
662 			} else if (city.getIs_in_zip()!=null && city.getIs_in_zip().size()>=1){
663 				address.setZipCode(city.getIs_in_zip().iterator().next());
664 			}
665 			address.setCountryCode(city.getCountry_code());
666 			address.setDependentLocality(city.getIs_in_place());
667 		}
668 	}
669 
670 	private boolean isIntersection(Address address) {
671 		return address.getStreetNameIntersection() != null;
672 	}
673 
674 	protected List<SolrResponseDto> findCitiesInText(String text, String countryCode) {
675 		return findInText(text, countryCode, null, com.gisgraphy.fulltext.Constants.CITY_AND_CITYSUBDIVISION_PLACETYPE);
676 	}
677 
678 	protected List<SolrResponseDto> findStreetInText(String text, String countryCode, Point point) {
679 		List<SolrResponseDto> streets = findInText(text, countryCode, point, com.gisgraphy.fulltext.Constants.STREET_PLACETYPE);
680 		Point location;
681 		if (point != null) {
682 			for (SolrResponseDto solrResponseDto : streets) {
683 				Double longitude = solrResponseDto.getLng();
684 				Double latitude = solrResponseDto.getLat();
685 				if (latitude != null && longitude != null) {
686 					location = GeolocHelper.createPoint(longitude.floatValue(), latitude.floatValue());
687 					Double distance = GeolocHelper.distance(location, point);
688 					solrResponseDto.setDistance(distance);
689 				}
690 			}
691 			Collections.sort(streets, comparator);
692 		}
693 		return streets;
694 	}
695 
696 	protected List<SolrResponseDto> findInText(String text, String countryCode, Point point, Class<?>[] placetypes) {
697 		if (isEmptyString(text)) {
698 			return new ArrayList<SolrResponseDto>();
699 		}
700 		Output output;
701 		if (placetypes != null && placetypes.length == 1 && placetypes[0] == Street.class) {
702 			output = MEDIUM_OUTPUT;
703 		} else {
704 			output = LONG_OUTPUT;
705 		}
706 		FulltextQuery query = new FulltextQuery(text, Pagination.DEFAULT_PAGINATION, output, placetypes, countryCode);
707 		query.withAllWordsRequired(false).withoutSpellChecking();
708 		if (point != null) {
709 			query.around(point);
710 			query.withRadius(ACCEPT_DISTANCE_BETWEEN_CITY_AND_STREET);
711 		}
712 		FulltextResultsDto results = fullTextSearchEngine.executeQuery(query);
713 		if (results.getResultsSize() >= 1) {
714 			return results.getResults();
715 		} else {
716 			return new ArrayList<SolrResponseDto>();
717 		}
718 	}
719 
720 	/**
721 	 * @param exactMatches
722 	 * @param aproximativeMatches
723 	 * @return a list of {@link SolrResponseDto} with
724 	 *         list1[0],list2[0],list1[1],list2[1],... it remove duplicates and
725 	 *         null
726 	 */
727 	protected List<SolrResponseDto> mergeSolrResponseDto(List<SolrResponseDto> exactMatches, List<SolrResponseDto> aproximativeMatches) {
728 		/*find common id and put them first
729 		retirer duplicate de exact (si street)
730 		retirer duplicate de approximate (si street)
731 		merger*/
732 		if (exactMatches == null || exactMatches.size() == 0) {
733 			if (aproximativeMatches == null) {
734 				return new ArrayList<SolrResponseDto>();
735 			} else {
736 				return aproximativeMatches;
737 			}
738 		} else if (aproximativeMatches == null || aproximativeMatches.size() == 0) {
739 			return exactMatches;
740 
741 		} else {
742 			List<SolrResponseDto> merged = new ArrayList<SolrResponseDto>();
743 			int maxSize = Math.max(exactMatches.size(), aproximativeMatches.size());
744 			for (int i = 0; i < maxSize; i++) {
745 				if (i < exactMatches.size() && !merged.contains(exactMatches.get(i)) && exactMatches.get(i) != null) {
746 					merged.add(exactMatches.get(i));
747 				}
748 				if (i < aproximativeMatches.size() && !merged.contains(aproximativeMatches.get(i)) && aproximativeMatches.get(i) != null) {
749 					merged.add(aproximativeMatches.get(i));
750 				}
751 			}
752 			return merged;
753 		}
754 	}
755 
756 	protected List<SolrResponseDto> findExactMatches(String text, String countryCode) {
757 		if (isEmptyString(text)) {
758 			return new ArrayList<SolrResponseDto>();
759 		}
760 		FulltextQuery query = new FulltextQuery(text, FIVE_RESULT_PAGINATION, LONG_OUTPUT, ADDRESSES_PLACETYPE, countryCode);
761 		query.withAllWordsRequired(true).withoutSpellChecking();
762 		FulltextResultsDto results = fullTextSearchEngine.executeQuery(query);
763 		if (results.getResultsSize() >= 1) {
764 			return results.getResults();
765 		} else {
766 			return new ArrayList<SolrResponseDto>();
767 		}
768 	}
769 	
770 	protected String findHouseNumber(String address,String countryCode){
771 		Matcher m = HOUSENUMBERPATTERN.matcher(address);
772 		if (m.find()) {
773 			String houseNumber = m.group();
774 			logger.info("found house number"+houseNumber+" in "+address);
775 			return houseNumber.trim();
776 		} else {
777 			logger.info("no house number found in "+address);
778 			return null;
779 		}
780 		
781 	}
782 
783 	@Autowired
784 	public void setAddressParser(IAddressParserService addressParser) {
785 		this.addressParser = addressParser;
786 	}
787 
788 	@Autowired
789 	public void setFullTextSearchEngine(IFullTextSearchEngine fullTextSearchEngine) {
790 		this.fullTextSearchEngine = fullTextSearchEngine;
791 	}
792 
793 	@Autowired
794 	public void setStatsUsageService(IStatsUsageService statsUsageService) {
795 		this.statsUsageService = statsUsageService;
796 	}
797 
798 	@Autowired
799 	public void setImporterConfig(ImporterConfig importerConfig) {
800 		this.importerConfig = importerConfig;
801 	}
802 
803 	@Autowired
804 	public void setGisgraphyConfig(GisgraphyConfig gisgraphyConfig) {
805 		this.gisgraphyConfig = gisgraphyConfig;
806 	}
807 
808 }