View Javadoc
1   /*******************************************************************************
2    *   Gisgraphy Project 
3    * 
4    *   This library is free software; you can redistribute it and/or
5    *   modify it under the terms of the GNU Lesser General Public
6    *   License as published by the Free Software Foundation; either
7    *   version 2.1 of the License, or (at your option) any later version.
8    * 
9    *   This library is distributed in the hope that it will be useful,
10   *   but WITHOUT ANY WARRANTY; without even the implied warranty of
11   *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12   *   Lesser General Public License for more details.
13   * 
14   *   You should have received a copy of the GNU Lesser General Public
15   *   License along with this library; if not, write to the Free Software
16   *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
17   * 
18   *  Copyright 2008  Gisgraphy project 
19   *  David Masclet <davidmasclet@gisgraphy.com>
20   *  
21   *  
22   *******************************************************************************/
23  /**
24   *
25   */
26  package com.gisgraphy.importer;
27  
28  import java.io.File;
29  import java.io.FileNotFoundException;
30  import java.io.IOException;
31  import java.util.ArrayList;
32  import java.util.List;
33  
34  import org.slf4j.Logger;
35  import org.slf4j.LoggerFactory;
36  import org.springframework.beans.factory.annotation.Autowired;
37  import org.springframework.beans.factory.annotation.Required;
38  
39  import com.gisgraphy.domain.valueobject.ImporterStatus;
40  import com.gisgraphy.domain.valueobject.NameValueDTO;
41  import com.gisgraphy.service.IInternationalisationService;
42  
43  /**
44   *  Base class to download files from a server
45   * 
46   * @author <a href="mailto:david.masclet@gisgraphy.com">David Masclet</a>
47   */
48  public abstract class AbstractFileRetriever implements IImporterProcessor {
49      
50     @Autowired
51      private IInternationalisationService internationalisationService;
52  
53      protected ImporterConfig importerConfig;
54  
55      protected String currentFileName = null;
56  
57      protected ImporterStatus status = ImporterStatus.WAITING;
58  
59      protected int fileIndex = 0;
60  
61      protected int numberOfFileToDownload = 0;
62  
63      protected String statusMessage = "";
64      
65      private long fileSizeToDownloadCached =0;
66      
67      
68      /**
69       * The logger
70       */
71      protected static final Logger logger = LoggerFactory
72  	    .getLogger(AbstractFileRetriever.class);
73  
74      /*
75       * (non-Javadoc)
76       * 
77       * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#process()
78       */
79      public void process() throws ImporterException {
80  	statusMessage = internationalisationService.getString("import.download.info");
81  	status = ImporterStatus.PROCESSING;
82  	try {
83  	    if (!shouldBeSkipped()) {
84  		if (importerConfig.isRetrieveFiles()){
85  		logger
86  			.info("DownloadFiles option is set to true, we will download and decompress files");
87  		downloadFiles();
88  		} else {
89  		    logger
90  			.info("DownloadFiles option is set to false, we won't download but decompress files");
91  		}
92  		statusMessage = internationalisationService.getString("import.extract.info");
93  		decompressFiles();
94  		this.status = ImporterStatus.PROCESSED ;
95  	    } else {
96  		this.status = ImporterStatus.SKIPPED;
97  		logger
98  			.info("DownloadFiles option is set to false, we will not download and decompress files");
99  	    }
100 	    statusMessage = "";
101 	} catch (Exception e) {
102 	    this.statusMessage = "error retrieving or decompres file : " + e.getMessage();
103 	    logger.error(statusMessage,e);
104 	    status = ImporterStatus.ERROR;
105 	    throw new ImporterException(statusMessage, e);
106 	} 
107 
108     }
109 
110     protected void downloadFiles() {
111 	List<String> downloadFileList = getFilesToDownload();
112 	this.numberOfFileToDownload = downloadFileList.size();
113 	for (String file : downloadFileList) {
114 	    this.fileIndex++;
115 	    this.currentFileName = file;
116 	    try {
117 		downloadFile(file);
118 	    } catch (FileNotFoundException e) {
119 		if (isFileNotFoundTolerant()){
120 		    logger.error(getDownloadBaseUrl()
121 		    + file+" can not be downloaded" );
122 		} else {
123 		 throw new RuntimeException(e);    
124 		}
125 		}
126 	}
127     }
128 
129     protected void downloadFile(String file) throws FileNotFoundException {
130 	ImporterHelper.download(getDownloadBaseUrl()
131 	    + file, getDownloadDirectory() + file);
132     }
133     
134     /**
135      * @return false if download files that doesn't exists on the remote server
136      *         should throw
137      */
138     public abstract boolean isFileNotFoundTolerant();
139 
140     /**
141      * @return A list of file to be download
142      */
143     abstract List<String> getFilesToDownload() ;
144 
145     /**
146      * @return true if the processor should Not be executed
147      */
148     public boolean shouldBeSkipped() {
149 	return importerConfig.isRetrieveFiles();
150     }
151 
152     /**
153      * Method to call if files must be decompress (untar or unzip)
154      * @throws IOException
155      */
156     public abstract void decompressFiles() throws IOException ;
157     
158     /**
159      * return an array of file that are to be decompressed
160      * @throws IOException
161      */
162     public abstract File[] getFilesToDecompress() throws IOException ;
163 
164     /** 
165      * @return The directory where the file should be downloaded
166      */
167     public abstract String getDownloadDirectory();
168 
169     /**
170      * @return the base URL from wich the file should be downloaded
171      */
172     public abstract String getDownloadBaseUrl() ;
173 
174    
175 
176     /*
177      * (non-Javadoc)
178      * 
179      * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#getReadFileLine()
180      */
181     public long getReadFileLine() {
182 	File file = new File( getDownloadDirectory() + this.currentFileName);
183 	if (!file.exists() || !file.isFile()){
184 	    return 0;
185 	}
186 	return new Long(file.length());
187     }
188 
189     /* (non-Javadoc)
190      * @see com.gisgraphy.domain.geoloc.importer.IImporterProcessor#getTotalReadLine()
191      */
192     public long getTotalReadLine() {
193 	if (ImporterStatus.SKIPPED.equals(status) || ImporterStatus.WAITING.equals(status)){
194 	    return 0;
195 	}
196 	List<String> downloadedFiles;
197 	try {
198 	    downloadedFiles = getFilesToDownload();
199 	} catch (Exception e) {
200 	    logger.error("Can not retrieve files to decompress to calculate alredy downloaded file "+ e.getMessage(),e);
201 	   return -1;
202 	}
203 	long cumulatedSize = 0;
204 	for (String filename : downloadedFiles) {
205 		File file =new File(getDownloadDirectory() + filename);
206 	    if (file.exists() && file.isFile()){
207 		cumulatedSize = cumulatedSize+file.length();
208 	    }
209 	}
210 	return cumulatedSize;
211     }
212 
213     /*
214      * (non-Javadoc)
215      * 
216      * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#getCurrentFile()
217      */
218     public String getCurrentFileName() {
219 	if (this.currentFileName != null) {
220 	    return this.currentFileName;
221 	}
222 	return "?";
223     }
224 
225     public long getNumberOfLinesToProcess() {
226 	if (fileSizeToDownloadCached==0 && importerConfig.isRetrieveFiles()){
227 	List<String> downloadFileList = getFilesToDownload();
228 	long cumulatedSize = 0;
229 	for (String file : downloadFileList) {
230 	    long size = ImporterHelper.getHttpFileSize((getDownloadBaseUrl()
231 		    + file));
232 	    if (size !=-1){
233 		cumulatedSize = cumulatedSize +size;
234 	    }
235 	}
236 	fileSizeToDownloadCached = cumulatedSize;
237 	}
238 	return fileSizeToDownloadCached;
239 	
240     }
241 
242     public ImporterStatus getStatus() {
243 	return this.status;
244     }
245 
246     public String getStatusMessage() {
247 	return this.statusMessage;
248     }
249 
250     /*
251      * (non-Javadoc)
252      * 
253      * @see com.gisgraphy.domain.geoloc.importer.IGeonamesProcessor#rollback()
254      */
255     public List<NameValueDTO<Integer>> rollback() {
256 	List<NameValueDTO<Integer>> deletedObjectInfo = new ArrayList<NameValueDTO<Integer>>();
257 	List<String> filesToImport = getFilesToDownload();
258 	int deleted = 0;
259 	for (String fileName : filesToImport) {
260 	    File file = new File(getDownloadDirectory() + fileName);
261 	    if (file.delete()) {
262 		logger
263 			.info("the files " + file.getName()
264 				+ " has been deleted");
265 		deleted++;
266 	    } else {
267 		logger.info("the files " + file.getName()
268 			+ " hasn't been deleted");
269 	    }
270 	}
271 	deletedObjectInfo.add(new NameValueDTO<Integer>("Downloaded files",
272 		deleted));
273 	resetStatus();
274 	return deletedObjectInfo;
275     }
276 
277     public void resetStatus() {
278 	currentFileName = null;
279 	status = ImporterStatus.WAITING;
280 	fileIndex = 0;
281 	numberOfFileToDownload = 0;
282 	statusMessage = "";
283     }
284     
285     /**
286      * @param importerConfig
287      *                The importerConfig to set
288      */
289     @Required
290     public void setImporterConfig(ImporterConfig importerConfig) {
291 	this.importerConfig = importerConfig;
292     }
293     
294     /**
295      * @param internationalisationService the internationalisationService to set
296      */
297     @Required
298     public void setInternationalisationService(IInternationalisationService internationalisationService) {
299         this.internationalisationService = internationalisationService;
300     }
301 
302 
303 }