View Javadoc
1   /*******************************************************************************
2    *   Gisgraphy Project 
3    * 
4    *   This library is free software; you can redistribute it and/or
5    *   modify it under the terms of the GNU Lesser General Public
6    *   License as published by the Free Software Foundation; either
7    *   version 2.1 of the License, or (at your option) any later version.
8    * 
9    *   This library is distributed in the hope that it will be useful,
10   *   but WITHOUT ANY WARRANTY; without even the implied warranty of
11   *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12   *   Lesser General Public License for more details.
13   * 
14   *   You should have received a copy of the GNU Lesser General Public
15   *   License along with this library; if not, write to the Free Software
16   *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
17   * 
18   *  Copyright 2008  Gisgraphy project 
19   *  David Masclet <davidmasclet@gisgraphy.com>
20   *  
21   *  
22   *******************************************************************************/
23  package com.gisgraphy.importer;
24  
25  import java.io.File;
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.logging.Level;
30  
31  import org.slf4j.Logger;
32  import org.slf4j.LoggerFactory;
33  import org.springframework.beans.factory.annotation.Autowired;
34  import org.springframework.beans.factory.annotation.Required;
35  
36  import com.gisgraphy.domain.geoloc.entity.GisFeature;
37  import com.gisgraphy.domain.repository.IDatabaseHelper;
38  import com.gisgraphy.domain.repository.IGisDao;
39  import com.gisgraphy.domain.repository.IImporterStatusListDao;
40  import com.gisgraphy.domain.repository.ISolRSynchroniser;
41  import com.gisgraphy.domain.valueobject.ImporterStatusDto;
42  import com.gisgraphy.fulltext.IsolrClient;
43  import com.gisgraphy.helper.FileHelper;
44  import com.gisgraphy.service.impl.StatsUsageServiceImpl;
45  
46  /**
47   * Do the importing stuff
48   * 
49   * @author <a href="mailto:david.masclet@gisgraphy.com">David Masclet</a>
50   */
51  public class ImporterManager implements IImporterManager {
52  
53      private List<IImporterProcessor> importers = null;
54  
55      private ImporterConfig importerConfig;
56  
57      @Autowired
58      IGisDao<? extends GisFeature>[] iDaos;
59  
60      private long startTime = 0;
61  
62      private long endTime = 0;
63  
64      private boolean inProgress = false;
65  
66  
67      private ISolRSynchroniser solRSynchroniser;
68  
69      IImporterStatusListDao importerStatusListDao;
70      
71      @Autowired
72      private IsolrClient solrClient;
73      
74      @Autowired
75      private IDatabaseHelper databaseHelper;
76  
77    
78  
79  
80      /**
81       * The logger
82       */
83      protected static final Logger logger = LoggerFactory
84  	    .getLogger(ImporterManager.class);
85  
86     
87  
88      /*
89       * (non-Javadoc)
90       * 
91       * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#importAll()
92       */
93      public synchronized void importAll() {
94  	if (this.inProgress == true) {
95  	    logger
96  		    .error("You can not run an import because an other one is in progress");
97  	    return;
98  	}
99  	try {
100 	    if (isAlreadyDone() == true) {
101 	        logger
102 	    	    .error("You can not run an import because an other has already been done, if you want to run an other import, you must reset all the database and the fulltext search engine first");
103 	        return;
104 	    }
105 	} catch (ImporterMetaDataException e1) {
106 	    throw new ImporterException(e1.getMessage(),e1);
107 	}
108 	this.startTime = System.currentTimeMillis();
109 	importerStatusListDao.delete();
110 	try {
111 	    solrClient.setSolRLogLevel(Level.WARNING);
112 	    logger.info("temporarily disabling stats");
113 	    StatsUsageServiceImpl.disabled=true;
114 	    this.inProgress = true;
115 	    for (IImporterProcessor importer : importers) {
116 		logger.info("will now process "
117 			+ importer.getClass().getSimpleName());
118 		importer.process();
119 	    }
120 	    logger.info("end of import");
121 	} finally {
122 		try {
123 			logger.info("re-enabling stats");
124 			StatsUsageServiceImpl.disabled=false;
125 			this.importerStatusListDao.saveOrUpdate(ComputeStatusDtoList());
126 		} catch (RuntimeException e) {
127 			logger.error("Can not save statusDtoList : " + e.getMessage(),e);
128 		}
129 		try {
130 			this.endTime = System.currentTimeMillis();
131 			this.inProgress = false;
132 			setAlreadyDone(true);
133 		} catch (Exception e) {
134 			logger.error("The import is done but we can not persist the already done status : "+e.getMessage(),e);
135 		}
136 	}
137     }
138 
139     /*
140      * (non-Javadoc)
141      * 
142      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#getStatusDtoList()
143      */
144     public List<ImporterStatusDto> getStatusDtoList() {
145 	try {
146 	    if (isInProgress()) {
147 		return ComputeStatusDtoList();
148 	    } else {
149 		return importerStatusListDao.get();
150 	    }
151 	} catch (RuntimeException e) {
152 	    logger.error("Can not retrieve or process statusDtoList : "
153 		    + e.getMessage(),e);
154 	    return new ArrayList<ImporterStatusDto>();
155 	}
156     }
157 
158     private List<ImporterStatusDto> ComputeStatusDtoList() {
159 	List<ImporterStatusDto> list = new ArrayList<ImporterStatusDto>();
160 	for (IImporterProcessor processor : importers) {
161 	    list.add(new ImporterStatusDto(processor));
162 	}
163 	return list;
164     }
165 
166     /*
167      * (non-Javadoc)
168      * 
169      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#getImporters()
170      */
171     public List<IImporterProcessor> getImporters() {
172 	return importers;
173     }
174 
175     /*
176      * (non-Javadoc)
177      * 
178      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#getImporterConfig()
179      */
180     public ImporterConfig getImporterConfig() {
181 	return importerConfig;
182     }
183 
184     /**
185      * @return the time the last import took. If the import is in progress,
186      *         returns the time it took from the beginning. If the import has
187      *         not been started yet return 0.
188      */
189     public long getTimeElapsed() {
190 	if (this.startTime == 0) {
191 	    return 0;
192 	} else {
193 	    if (this.inProgress) {
194 		return (System.currentTimeMillis() - startTime) / 1000;
195 	    } else {
196 		return (this.endTime - this.startTime) / 1000;
197 	    }
198 	}
199     }
200 
201     /*
202      * (non-Javadoc)
203      * 
204      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#getFormatedTimeElapsed()
205      */
206     public String getFormatedTimeElapsed() {
207 	return ImporterHelper.formatSeconds(getTimeElapsed());
208     }
209 
210     /*
211      * (non-Javadoc)
212      * 
213      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#isInProgress()
214      */
215     public boolean isInProgress() {
216 	return inProgress;
217     }
218 
219     /*
220      * (non-Javadoc)
221      * 
222      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#isAlreadyDone()
223      */
224     public boolean isAlreadyDone() throws ImporterMetaDataException {
225 	if (!new File(importerConfig.getImporterMetadataDirectoryPath()).exists()){
226 	    throw new ImporterMetaDataException("can not determine if the import is already done because the directory where the metadata of the importer "+importerConfig.getImporterMetadataDirectoryPath()+"  doesn't exists");
227 	}
228 	return !new File(importerConfig.getAlreadyDoneFilePath()).exists();
229     }
230 
231     
232     
233     /*
234      * (non-Javadoc)
235      * 
236      * @see com.gisgraphy.domain.geoloc.importer.IImporterManager#resetImport()
237      */
238     public List<String> resetImport() throws Exception {
239 	solrClient.setSolRLogLevel(Level.WARNING);
240 	List<String> warningAndErrorMessage = new ArrayList<String>();
241 
242 	    File tempDir = FileHelper.createTempDir(this.getClass().getSimpleName());
243 	    File fileToCreateTablesToReRunImport = new File(tempDir.getAbsolutePath() + System.getProperty("file.separator") + "createTables.sql");
244 	    File fileToDropTablesToReRunImport = new File(tempDir.getAbsolutePath() + System.getProperty("file.separator") + "dropTables.sql");
245 	    databaseHelper.generateSqlDropSchemaFileToRerunImport(fileToDropTablesToReRunImport);
246 	    databaseHelper.generateSQLCreationSchemaFileToRerunImport(fileToCreateTablesToReRunImport);
247 	    
248 	    List<String> dropErrorMessage = databaseHelper.execute(fileToDropTablesToReRunImport, true);
249 	    List<String> creationErrorMessage = databaseHelper.execute(fileToCreateTablesToReRunImport, true);
250 	    warningAndErrorMessage.addAll(dropErrorMessage);
251 	    warningAndErrorMessage.addAll(creationErrorMessage);
252 	    
253 	    resetFullTextSearchEngine();
254 	    setAlreadyDone(false);
255 	    for (IImporterProcessor importer :importers){
256 		importer.resetStatus();
257 	    }
258 	    importerStatusListDao.delete();
259 	    this.inProgress = false;
260 	    tempDir.delete();
261 	return warningAndErrorMessage;
262 
263     }
264 
265    /**
266      * 
267      */
268     private void resetFullTextSearchEngine() {
269 	logger.info("will reset fulltext search engine");
270 	solRSynchroniser.deleteAll();
271 	logger.info("fulltext search engine has been reset");
272 	logger.info("end of reset");
273     }
274 
275     /**
276      * @param solRSynchroniser
277      *                the solRSynchroniser to set
278      */
279     @Required
280     public void setSolRSynchroniser(ISolRSynchroniser solRSynchroniser) {
281 	this.solRSynchroniser = solRSynchroniser;
282     }
283 
284     /**
285      * @param importerConfig
286      *                The {@link ImporterConfig} to set
287      */
288     @Required
289     public void setImporterConfig(ImporterConfig importerConfig) {
290 	this.importerConfig = importerConfig;
291     }
292 
293     /**
294      * @param importers
295      *                The importers to process
296      */
297     @Required
298     public void setImporters(List<IImporterProcessor> importers) {
299 	this.importers = importers;
300     }
301 
302     /**
303      * @param daos
304      *                the iDaos to set
305      */
306     public void setIDaos(IGisDao<? extends GisFeature>[] daos) {
307 	iDaos = daos;
308     }
309 
310     private void setAlreadyDone(boolean alreadyDone) {
311 	importerConfig.createImporterMetadataDirIfItDoesnTExist();
312 	File alreadyDoneFile = new File(importerConfig.getAlreadyDoneFilePath());
313 	if (alreadyDone == false) {
314 	    if (!alreadyDoneFile.exists()) {
315 		try {
316 		    boolean created = alreadyDoneFile.createNewFile();
317 		    if (created == false) {
318 			throw new ImporterException("Can not change the already done status to " + alreadyDone);
319 		    }
320 		} catch (IOException e) {
321 		    throw new ImporterException("Can not change the already done status to " + alreadyDone + " : " + e.getMessage(),e);
322 		}
323 	    }
324 
325 	} else {
326 	    if (alreadyDoneFile.exists()) {
327 		boolean deleted = alreadyDoneFile.delete();
328 		if (deleted == false) {
329 		    throw new ImporterException("Can not change the already done status to " + alreadyDone);
330 		}
331 	    }
332 
333 	}
334     }
335     
336    
337     
338     
339     /**
340      * @param importerStatusListDao
341      *                the importerStatusListDao to set
342      */
343     @Required
344     public void setImporterStatusListDao(
345 	    IImporterStatusListDao importerStatusListDao) {
346 	this.importerStatusListDao = importerStatusListDao;
347     }
348 
349     /**
350      * @param solrClient the solrClient to set
351      */
352     @Required
353     public void setSolrClient(IsolrClient solrClient) {
354         this.solrClient = solrClient;
355     }
356 
357     /**
358      * @param databaseHelper the databaseHelper to set
359      */
360     @Required
361     public void setDatabaseHelper(IDatabaseHelper databaseHelper) {
362         this.databaseHelper = databaseHelper;
363     }
364     
365     
366 
367 }