GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_port.h"
36 #include "cpl_string.h"
37 #include "cpl_vsil_curl_priv.h"
38 #include "cpl_mem_cache.h"
39 
40 #include <curl/curl.h>
41 
42 #include <set>
43 #include <map>
44 #include <memory>
45 
47 
48 // 7.18.1
49 #if LIBCURL_VERSION_NUM >= 0x071201
50 #define HAVE_CURLINFO_REDIRECT_URL
51 #endif
52 
53 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
54 
55 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
56  const char * const* papszOptions);
57 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
58  struct curl_slist* poSrcToDestroy );
59 
60 namespace cpl {
61 
62 typedef enum
63 {
64  EXIST_UNKNOWN = -1,
65  EXIST_NO,
66  EXIST_YES,
67 } ExistStatus;
68 
69 class FileProp
70 {
71  public:
72  ExistStatus eExists = EXIST_UNKNOWN;
73  vsi_l_offset fileSize = 0;
74  time_t mTime = 0;
75  time_t nExpireTimestampLocal = 0;
76  CPLString osRedirectURL{};
77  bool bHasComputedFileSize = false;
78  bool bIsDirectory = false;
79  bool bS3LikeRedirect = false;
80  CPLString ETag{};
81 };
82 
83 typedef struct
84 {
85  bool bGotFileList = false;
86  CPLStringList oFileList{}; /* only file name without path */
87 } CachedDirList;
88 
89 typedef struct
90 {
91  char* pBuffer;
92  size_t nSize;
93  bool bIsHTTP;
94  bool bIsInHeader;
95  bool bMultiRange;
96  vsi_l_offset nStartOffset;
97  vsi_l_offset nEndOffset;
98  int nHTTPCode;
99  vsi_l_offset nContentLength;
100  bool bFoundContentRange;
101  bool bError;
102  bool bDownloadHeaderOnly;
103  bool bDetectRangeDownloadingError;
104  GIntBig nTimestampDate; // Corresponds to Date: header field
105 
106  VSILFILE *fp;
107  VSICurlReadCbkFunc pfnReadCbk;
108  void *pReadCbkUserData;
109  bool bInterrupted;
110 } WriteFuncStruct;
111 
112 /************************************************************************/
113 /* VSICurlFilesystemHandler */
114 /************************************************************************/
115 
116 class VSICurlHandle;
117 
118 class VSICurlFilesystemHandler : public VSIFilesystemHandler
119 {
120  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
121 
122  struct FilenameOffsetPair
123  {
124  std::string filename_;
125  vsi_l_offset offset_;
126 
127  FilenameOffsetPair(const std::string& filename,
128  vsi_l_offset offset) :
129  filename_(filename), offset_(offset) {}
130 
131  bool operator==(const FilenameOffsetPair& other) const
132  {
133  return filename_ == other.filename_ &&
134  offset_ == other.offset_;
135  }
136  };
137  struct FilenameOffsetPairHasher
138  {
139  std::size_t operator()(const FilenameOffsetPair& k) const
140  {
141  return std::hash<std::string>()(k.filename_) ^
142  std::hash<vsi_l_offset>()(k.offset_);
143  }
144  };
145 
146  using RegionCacheType =
147  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
148  lru11::NullLock,
149  std::unordered_map<
150  FilenameOffsetPair,
151  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
152  std::shared_ptr<std::string>>>::iterator,
153  FilenameOffsetPairHasher>>;
154 
155  RegionCacheType oRegionCache;
156 
157  lru11::Cache<std::string, FileProp> oCacheFileProp;
158 
159  int nCachedFilesInDirList = 0;
160  lru11::Cache<std::string, CachedDirList> oCacheDirList;
161 
162  char** ParseHTMLFileList(const char* pszFilename,
163  int nMaxFiles,
164  char* pszData,
165  bool* pbGotFileList);
166 
167 protected:
168  CPLMutex *hMutex = nullptr;
169 
170  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
171  virtual char** GetFileList(const char *pszFilename,
172  int nMaxFiles,
173  bool* pbGotFileList);
174 
175  void RegisterEmptyDir( const CPLString& osDirname );
176 
177  bool AnalyseS3FileList( const CPLString& osBaseURL,
178  const char* pszXML,
179  CPLStringList& osFileList,
180  int nMaxFiles,
181  bool bIgnoreGlacierStorageClass,
182  bool& bIsTruncated );
183 
184  void AnalyseSwiftFileList( const CPLString& osBaseURL,
185  const CPLString& osPrefix,
186  const char* pszJson,
187  CPLStringList& osFileList,
188  int nMaxFilesThisQuery,
189  int nMaxFiles,
190  bool& bIsTruncated,
191  CPLString& osNextMarker );
192 
193  static const char* GetOptionsStatic();
194 
195  static bool IsAllowedFilename( const char* pszFilename );
196 
197 public:
198  VSICurlFilesystemHandler();
199  ~VSICurlFilesystemHandler() override;
200 
201  VSIVirtualHandle *Open( const char *pszFilename,
202  const char *pszAccess,
203  bool bSetError ) override;
204 
205  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
206  int nFlags ) override;
207  int Unlink( const char *pszFilename ) override;
208  int Rename( const char *oldpath, const char *newpath ) override;
209  int Mkdir( const char *pszDirname, long nMode ) override;
210  int Rmdir( const char *pszDirname ) override;
211  char **ReadDir( const char *pszDirname ) override
212  { return ReadDirEx(pszDirname, 0); }
213  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
214 
215  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
216  override { return true; }
217 
218  const char* GetActualURL(const char* pszFilename) override;
219 
220  const char* GetOptions() override;
221 
222  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
223  bool* pbGotFileList );
224  void InvalidateDirContent( const char *pszDirname );
225 
226  virtual CPLString GetFSPrefix() { return "/vsicurl/"; }
227  virtual bool AllowCachedDataFor(const char* pszFilename);
228 
229  std::shared_ptr<std::string> GetRegion( const char* pszURL,
230  vsi_l_offset nFileOffsetStart );
231 
232  void AddRegion( const char* pszURL,
233  vsi_l_offset nFileOffsetStart,
234  size_t nSize,
235  const char *pData );
236 
237  bool GetCachedFileProp( const char* pszURL,
238  FileProp& oFileProp );
239  void SetCachedFileProp( const char* pszURL,
240  const FileProp& oFileProp );
241  void InvalidateCachedData( const char* pszURL );
242 
243  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
244 
245  virtual void ClearCache();
246  virtual void PartialClearCache(const char* pszFilename);
247 
248 
249  bool GetCachedDirList( const char* pszURL,
250  CachedDirList& oCachedDirList );
251  void SetCachedDirList( const char* pszURL,
252  const CachedDirList& oCachedDirList );
253  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
254 
255  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
256 };
257 
258 /************************************************************************/
259 /* VSICurlHandle */
260 /************************************************************************/
261 
262 class VSICurlHandle : public VSIVirtualHandle
263 {
264  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
265 
266  protected:
267  VSICurlFilesystemHandler* poFS = nullptr;
268 
269  bool m_bCached = true;
270 
271  FileProp oFileProp{};
272 
273  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
274  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
275 
276  char **m_papszHTTPOptions = nullptr;
277 
278  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
279  int nBlocksToDownload = 1;
280 
281  bool bStopOnInterruptUntilUninstall = false;
282  bool bInterrupted = false;
283  VSICurlReadCbkFunc pfnReadCbk = nullptr;
284  void *pReadCbkUserData = nullptr;
285 
286  int m_nMaxRetry = 0;
287  double m_dfRetryDelay = 0.0;
288 
289  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
290  const int nBlocks,
291  const char* pBuffer,
292  size_t nSize );
293 
294  private:
295 
296  vsi_l_offset curOffset = 0;
297 
298  bool bEOF = false;
299 
300  virtual bool DownloadRegion(vsi_l_offset startOffset, int nBlocks);
301 
302  bool m_bUseHead = false;
303 
304  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
305  const vsi_l_offset* panOffsets,
306  const size_t* panSizes );
307  CPLString GetRedirectURLIfValid(bool& bHasExpired);
308 
309  protected:
310  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
311  const struct curl_slist* /* psExistingHeaders */)
312  { return nullptr; }
313  virtual bool AllowAutomaticRedirection() { return true; }
314  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
315  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
316  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
317  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
318  void SetURL(const char* pszURL);
319 
320  public:
321 
322  VSICurlHandle( VSICurlFilesystemHandler* poFS,
323  const char* pszFilename,
324  const char* pszURLIn = nullptr );
325  ~VSICurlHandle() override;
326 
327  int Seek( vsi_l_offset nOffset, int nWhence ) override;
328  vsi_l_offset Tell() override;
329  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
330  int ReadMultiRange( int nRanges, void ** ppData,
331  const vsi_l_offset* panOffsets,
332  const size_t* panSizes ) override;
333  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
334  int Eof() override;
335  int Flush() override;
336  int Close() override;
337 
338  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
339  vsi_l_offset GetFileSize() { return GetFileSize(false); }
340  virtual vsi_l_offset GetFileSize( bool bSetError );
341  bool Exists( bool bSetError );
342  bool IsDirectory() const { return oFileProp.bIsDirectory; }
343  time_t GetMTime() const { return oFileProp.mTime; }
344 
345  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
346  void* pfnUserData,
347  int bStopOnInterruptUntilUninstall );
348  int UninstallReadCbk();
349 
350  const char *GetURL() const { return m_pszURL; }
351 };
352 
353 /************************************************************************/
354 /* IVSIS3LikeFSHandler */
355 /************************************************************************/
356 
357 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
358 {
359  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
360 
361  protected:
362  char** GetFileList( const char *pszFilename,
363  int nMaxFiles,
364  bool* pbGotFileList ) override;
365 
366  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
367  const char* pszURI, bool bAllowNoObject) = 0;
368 
369  IVSIS3LikeFSHandler() = default;
370 
371  public:
372  int Unlink( const char *pszFilename ) override;
373  int Mkdir( const char *pszDirname, long nMode ) override;
374  int Rmdir( const char *pszDirname ) override;
375  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
376  int nFlags ) override;
377 
378  virtual int DeleteObject( const char *pszFilename );
379 
380  virtual const char* GetDebugKey() const = 0;
381 
382  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
383  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
384 
385  bool Sync( const char* pszSource, const char* pszTarget,
386  const char* const * papszOptions,
387  GDALProgressFunc pProgressFunc,
388  void *pProgressData,
389  char*** ppapszOutputs ) override;
390 
391  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
392  const char* const *papszOptions) override;
393 };
394 
395 /************************************************************************/
396 /* IVSIS3LikeHandle */
397 /************************************************************************/
398 
399 class IVSIS3LikeHandle: public VSICurlHandle
400 {
401  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
402 
403  protected:
404  bool UseLimitRangeGetInsteadOfHead() override { return true; }
405  bool IsDirectoryFromExists( const char* pszVerb,
406  int response_code ) override
407  {
408  // A bit dirty, but on S3, a GET on a existing directory returns a 416
409  return response_code == 416 && EQUAL(pszVerb, "GET") &&
410  CPLString(m_pszURL).back() == '/';
411  }
412  void ProcessGetFileSizeResult( const char* pszContent ) override
413  {
414  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
415  }
416 
417  public:
418  IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
419  const char* pszFilename,
420  const char* pszURLIn = nullptr ) :
421  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
422  ~IVSIS3LikeHandle() override {}
423 };
424 
425 /************************************************************************/
426 /* VSIS3WriteHandle */
427 /************************************************************************/
428 
429 class VSIS3WriteHandle final : public VSIVirtualHandle
430 {
431  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
432 
433  IVSIS3LikeFSHandler *m_poFS = nullptr;
434  CPLString m_osFilename{};
435  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
436  bool m_bUseChunked = false;
437 
438  vsi_l_offset m_nCurOffset = 0;
439  int m_nBufferOff = 0;
440  int m_nBufferSize = 0;
441  int m_nBufferOffReadCallback = 0;
442  bool m_bClosed = false;
443  GByte *m_pabyBuffer = nullptr;
444  CPLString m_osUploadID{};
445  int m_nPartNumber = 0;
446  std::vector<CPLString> m_aosEtags{};
447  CPLString m_osXML{};
448  int m_nOffsetInXML = 0;
449  bool m_bError = false;
450 
451  CURLM *m_hCurlMulti = nullptr;
452  CURL *m_hCurl = nullptr;
453  const void *m_pBuffer = nullptr;
454  CPLString m_osCurlErrBuf{};
455  size_t m_nChunkedBufferOff = 0;
456  size_t m_nChunkedBufferSize = 0;
457 
458  static size_t ReadCallBackBuffer( char *buffer, size_t size,
459  size_t nitems, void *instream );
460  bool InitiateMultipartUpload();
461  bool UploadPart();
462  static size_t ReadCallBackXML( char *buffer, size_t size,
463  size_t nitems, void *instream );
464  bool CompleteMultipart();
465  bool AbortMultipart();
466  bool DoSinglePartPUT();
467 
468  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
469  size_t nitems, void *instream );
470  size_t WriteChunked( const void *pBuffer,
471  size_t nSize, size_t nMemb );
472  int FinishChunkedTransfer();
473 
474  void InvalidateParentDirectory();
475 
476  public:
477  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
478  const char* pszFilename,
479  IVSIS3LikeHandleHelper* poS3HandleHelper,
480  bool bUseChunked );
481  ~VSIS3WriteHandle() override;
482 
483  int Seek( vsi_l_offset nOffset, int nWhence ) override;
484  vsi_l_offset Tell() override;
485  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
486  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
487  int Eof() override;
488  int Close() override;
489 
490  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
491 };
492 
493 /************************************************************************/
494 /* VSIAppendWriteHandle */
495 /************************************************************************/
496 
497 class VSIAppendWriteHandle : public VSIVirtualHandle
498 {
499  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
500 
501  protected:
502 
503  VSICurlFilesystemHandler* m_poFS = nullptr;
504  CPLString m_osFSPrefix{};
505  CPLString m_osFilename{};
506 
507  vsi_l_offset m_nCurOffset = 0;
508  int m_nBufferOff = 0;
509  int m_nBufferSize = 0;
510  int m_nBufferOffReadCallback = 0;
511  bool m_bClosed = false;
512  GByte *m_pabyBuffer = nullptr;
513  bool m_bError = false;
514 
515  static size_t ReadCallBackBuffer( char *buffer, size_t size,
516  size_t nitems, void *instream );
517  virtual bool Send(bool bIsLastBlock) = 0;
518 
519  public:
520  VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
521  const char* pszFSPrefix,
522  const char* pszFilename,
523  int nChunkSize );
524  virtual ~VSIAppendWriteHandle();
525 
526  int Seek( vsi_l_offset nOffset, int nWhence ) override;
527  vsi_l_offset Tell() override;
528  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
529  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
530  int Eof() override;
531  int Close() override;
532 
533  bool IsOK() { return m_pabyBuffer != nullptr; }
534 };
535 
536 int VSICURLGetDownloadChunkSize();
537 
538 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
539  VSILFILE *fp,
540  VSICurlReadCbkFunc pfnReadCbk,
541  void *pReadCbkUserData );
542 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
543  size_t nmemb, void *req );
544 void MultiPerform(CURLM* hCurlMultiHandle,
545  CURL* hEasyHandle = nullptr);
546 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
547 
548 } // namespace cpl
549 
551 
552 #endif // HAVE_CURL
553 
554 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
Definition: cpl_conv.h:367
Core portability definitions for CPL.
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Convenient string class based on std::string.
Definition: cpl_string.h:329
Various convenience functions for working with strings and string lists.
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:192
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:561
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:438
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:307
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:989
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142

Generated for GDAL by doxygen 1.8.8.