29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
37 #include "cpl_vsil_curl_priv.h"
38 #include "cpl_mem_cache.h"
40 #include <curl/curl.h>
49 #if LIBCURL_VERSION_NUM >= 0x071201
50 #define HAVE_CURLINFO_REDIRECT_URL
53 void VSICurlStreamingClearCache(
void );
55 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
56 const char *
const* papszOptions);
57 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
58 struct curl_slist* poSrcToDestroy );
72 ExistStatus eExists = EXIST_UNKNOWN;
75 time_t nExpireTimestampLocal = 0;
77 bool bHasComputedFileSize =
false;
78 bool bIsDirectory =
false;
79 bool bS3LikeRedirect =
false;
85 bool bGotFileList =
false;
91 char* pBuffer =
nullptr;
94 bool bIsInHeader =
false;
95 bool bMultiRange =
false;
100 bool bFoundContentRange =
false;
102 bool bDownloadHeaderOnly =
false;
103 bool bDetectRangeDownloadingError =
false;
107 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
108 void *pReadCbkUserData =
nullptr;
109 bool bInterrupted =
false;
111 #if LIBCURL_VERSION_NUM < 0x073600
115 bool bIsProxyConnectHeader =
false;
125 class VSICurlFilesystemHandler :
public VSIFilesystemHandler
129 struct FilenameOffsetPair
131 std::string filename_;
134 FilenameOffsetPair(
const std::string& filename,
136 filename_(filename), offset_(offset) {}
138 bool operator==(
const FilenameOffsetPair& other)
const
140 return filename_ == other.filename_ &&
141 offset_ == other.offset_;
144 struct FilenameOffsetPairHasher
146 std::size_t operator()(
const FilenameOffsetPair& k)
const
148 return std::hash<std::string>()(k.filename_) ^
149 std::hash<vsi_l_offset>()(k.offset_);
153 using RegionCacheType =
154 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
158 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
159 std::shared_ptr<std::string>>>::iterator,
160 FilenameOffsetPairHasher>>;
162 RegionCacheType oRegionCache;
164 lru11::Cache<std::string, FileProp> oCacheFileProp;
166 int nCachedFilesInDirList = 0;
167 lru11::Cache<std::string, CachedDirList> oCacheDirList;
169 char** ParseHTMLFileList(
const char* pszFilename,
172 bool* pbGotFileList);
175 CPLMutex *hMutex =
nullptr;
177 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
178 virtual char** GetFileList(
const char *pszFilename,
180 bool* pbGotFileList);
182 void RegisterEmptyDir(
const CPLString& osDirname );
184 bool AnalyseS3FileList(
const CPLString& osBaseURL,
188 bool bIgnoreGlacierStorageClass,
189 bool& bIsTruncated );
191 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
195 int nMaxFilesThisQuery,
200 static const char* GetOptionsStatic();
202 static bool IsAllowedFilename(
const char* pszFilename );
205 VSICurlFilesystemHandler();
206 ~VSICurlFilesystemHandler()
override;
209 const char *pszAccess,
210 bool bSetError )
override;
212 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
213 int nFlags )
override;
214 int Unlink(
const char *pszFilename )
override;
215 int Rename(
const char *oldpath,
const char *newpath )
override;
216 int Mkdir(
const char *pszDirname,
long nMode )
override;
217 int Rmdir(
const char *pszDirname )
override;
218 char **ReadDir(
const char *pszDirname )
override
219 {
return ReadDirEx(pszDirname, 0); }
220 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
222 int HasOptimizedReadMultiRange(
const char* )
223 override {
return true; }
225 const char* GetActualURL(
const char* pszFilename)
override;
227 const char* GetOptions()
override;
229 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
230 bool* pbGotFileList );
231 void InvalidateDirContent(
const char *pszDirname );
233 virtual CPLString GetFSPrefix() {
return "/vsicurl/"; }
234 virtual bool AllowCachedDataFor(
const char* pszFilename);
236 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
239 void AddRegion(
const char* pszURL,
244 bool GetCachedFileProp(
const char* pszURL,
245 FileProp& oFileProp );
246 void SetCachedFileProp(
const char* pszURL,
247 const FileProp& oFileProp );
248 void InvalidateCachedData(
const char* pszURL );
250 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
252 virtual void ClearCache();
253 virtual void PartialClearCache(
const char* pszFilename);
256 bool GetCachedDirList(
const char* pszURL,
257 CachedDirList& oCachedDirList );
258 void SetCachedDirList(
const char* pszURL,
259 const CachedDirList& oCachedDirList );
260 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
274 VSICurlFilesystemHandler* poFS =
nullptr;
276 bool m_bCached = true;
278 FileProp oFileProp{};
281 char* m_pszURL =
nullptr;
283 char **m_papszHTTPOptions =
nullptr;
286 int nBlocksToDownload = 1;
288 bool bStopOnInterruptUntilUninstall =
false;
289 bool bInterrupted =
false;
290 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
291 void *pReadCbkUserData =
nullptr;
294 double m_dfRetryDelay = 0.0;
296 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
307 virtual bool DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
309 bool m_bUseHead =
false;
311 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
313 const size_t* panSizes );
314 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
317 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
318 const struct curl_slist* )
320 virtual bool AllowAutomaticRedirection() {
return true; }
321 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
322 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
323 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
324 virtual void ProcessGetFileSizeResult(
const char* ) {}
325 void SetURL(
const char* pszURL);
329 VSICurlHandle( VSICurlFilesystemHandler* poFS,
330 const char* pszFilename,
331 const char* pszURLIn =
nullptr );
332 ~VSICurlHandle()
override;
336 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
337 int ReadMultiRange(
int nRanges,
void ** ppData,
339 const size_t* panSizes )
override;
340 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
342 int Flush()
override;
343 int Close()
override;
345 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
346 vsi_l_offset GetFileSize() {
return GetFileSize(
false); }
348 bool Exists(
bool bSetError );
349 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
350 time_t GetMTime()
const {
return oFileProp.mTime; }
352 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
354 int bStopOnInterruptUntilUninstall );
355 int UninstallReadCbk();
357 const char *GetURL()
const {
return m_pszURL; }
364 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandler
369 char** GetFileList( const
char *pszFilename,
371 bool* pbGotFileList ) override;
373 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
374 const
char* pszURI,
bool bAllowNoObject) = 0;
376 IVSIS3LikeFSHandler() = default;
379 int Unlink( const
char *pszFilename ) override;
380 int Mkdir( const
char *pszDirname,
long nMode ) override;
381 int Rmdir( const
char *pszDirname ) override;
382 int Stat( const
char *pszFilename,
VSIStatBufL *pStatBuf,
383 int nFlags ) override;
385 virtual
int DeleteObject( const
char *pszFilename );
387 virtual const
char* GetDebugKey() const = 0;
389 virtual
void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
390 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
392 bool Sync(
const char* pszSource,
const char* pszTarget,
393 const char*
const * papszOptions,
394 GDALProgressFunc pProgressFunc,
396 char*** ppapszOutputs )
override;
398 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
399 const char*
const *papszOptions)
override;
406 class IVSIS3LikeHandle:
public VSICurlHandle
411 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
412 bool IsDirectoryFromExists(
const char* pszVerb,
413 int response_code )
override
416 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
419 void ProcessGetFileSizeResult(
const char* pszContent )
override
421 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
425 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
426 const char* pszFilename,
427 const char* pszURLIn =
nullptr ) :
428 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
429 ~IVSIS3LikeHandle()
override {}
440 IVSIS3LikeFSHandler *m_poFS =
nullptr;
442 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
443 bool m_bUseChunked =
false;
446 int m_nBufferOff = 0;
447 int m_nBufferSize = 0;
448 int m_nBufferOffReadCallback = 0;
449 bool m_bClosed =
false;
450 GByte *m_pabyBuffer =
nullptr;
452 int m_nPartNumber = 0;
453 std::vector<CPLString> m_aosEtags{};
455 int m_nOffsetInXML = 0;
456 bool m_bError =
false;
458 CURLM *m_hCurlMulti =
nullptr;
459 CURL *m_hCurl =
nullptr;
460 const void *m_pBuffer =
nullptr;
462 size_t m_nChunkedBufferOff = 0;
463 size_t m_nChunkedBufferSize = 0;
466 double m_dfRetryDelay = 0.0;
467 WriteFuncStruct m_sWriteFuncHeaderData{};
469 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
470 size_t nitems,
void *instream );
471 bool InitiateMultipartUpload();
473 static size_t ReadCallBackXML(
char *buffer,
size_t size,
474 size_t nitems,
void *instream );
475 bool CompleteMultipart();
476 bool AbortMultipart();
477 bool DoSinglePartPUT();
479 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
480 size_t nitems,
void *instream );
481 size_t WriteChunked(
const void *pBuffer,
482 size_t nSize,
size_t nMemb );
483 int FinishChunkedTransfer();
485 void InvalidateParentDirectory();
488 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
489 const char* pszFilename,
490 IVSIS3LikeHandleHelper* poS3HandleHelper,
492 ~VSIS3WriteHandle()
override;
496 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
497 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
499 int Close()
override;
501 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
514 VSICurlFilesystemHandler* m_poFS =
nullptr;
519 int m_nBufferOff = 0;
520 int m_nBufferSize = 0;
521 int m_nBufferOffReadCallback = 0;
522 bool m_bClosed =
false;
523 GByte *m_pabyBuffer =
nullptr;
524 bool m_bError =
false;
526 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
527 size_t nitems,
void *instream );
528 virtual bool Send(
bool bIsLastBlock) = 0;
531 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
532 const char* pszFSPrefix,
533 const char* pszFilename,
535 virtual ~VSIAppendWriteHandle();
539 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
540 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
542 int Close()
override;
544 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
547 int VSICURLGetDownloadChunkSize();
549 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
551 VSICurlReadCbkFunc pfnReadCbk,
552 void *pReadCbkUserData );
553 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
554 size_t nmemb,
void *req );
555 void MultiPerform(CURLM* hCurlMultiHandle,
556 CURL* hEasyHandle =
nullptr);
557 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
565 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
Definition: cpl_conv.h:367
Core portability definitions for CPL.
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Convenient string class based on std::string.
Definition: cpl_string.h:329
Various convenience functions for working with strings and string lists.
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:192
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:561
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:438
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:307
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:989
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142