GDAL
cpl_vsil_curl_class.h
1/******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30#define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32#ifdef HAVE_CURL
33
34#include "cpl_aws.h"
35#include "cpl_port.h"
36#include "cpl_string.h"
37#include "cpl_vsil_curl_priv.h"
38#include "cpl_mem_cache.h"
39
40#include <curl/curl.h>
41
42#include <set>
43#include <map>
44#include <memory>
45
47
48// 7.18.1
49#if LIBCURL_VERSION_NUM >= 0x071201
50#define HAVE_CURLINFO_REDIRECT_URL
51#endif
52
53void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
54
55struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
56 const char * const* papszOptions);
57struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
58 struct curl_slist* poSrcToDestroy );
59
60namespace cpl {
61
62typedef enum
63{
64 EXIST_UNKNOWN = -1,
65 EXIST_NO,
66 EXIST_YES,
67} ExistStatus;
68
69class FileProp
70{
71 public:
72 ExistStatus eExists = EXIST_UNKNOWN;
73 vsi_l_offset fileSize = 0;
74 time_t mTime = 0;
75 time_t nExpireTimestampLocal = 0;
76 CPLString osRedirectURL{};
77 bool bHasComputedFileSize = false;
78 bool bIsDirectory = false;
79 bool bS3LikeRedirect = false;
80 CPLString ETag{};
81};
82
83typedef struct
84{
85 bool bGotFileList = false;
86 CPLStringList oFileList{}; /* only file name without path */
87} CachedDirList;
88
89typedef struct
90{
91 char* pBuffer;
92 size_t nSize;
93 bool bIsHTTP;
94 bool bIsInHeader;
95 bool bMultiRange;
96 vsi_l_offset nStartOffset;
97 vsi_l_offset nEndOffset;
98 int nHTTPCode;
99 vsi_l_offset nContentLength;
100 bool bFoundContentRange;
101 bool bError;
102 bool bDownloadHeaderOnly;
103 bool bDetectRangeDownloadingError;
104 GIntBig nTimestampDate; // Corresponds to Date: header field
105
106 VSILFILE *fp;
107 VSICurlReadCbkFunc pfnReadCbk;
108 void *pReadCbkUserData;
109 bool bInterrupted;
110} WriteFuncStruct;
111
112/************************************************************************/
113/* VSICurlFilesystemHandler */
114/************************************************************************/
115
116class VSICurlHandle;
117
118class VSICurlFilesystemHandler : public VSIFilesystemHandler
119{
120 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
121
122 struct FilenameOffsetPair
123 {
124 std::string filename_;
125 vsi_l_offset offset_;
126
127 FilenameOffsetPair(const std::string& filename,
128 vsi_l_offset offset) :
129 filename_(filename), offset_(offset) {}
130
131 bool operator==(const FilenameOffsetPair& other) const
132 {
133 return filename_ == other.filename_ &&
134 offset_ == other.offset_;
135 }
136 };
137 struct FilenameOffsetPairHasher
138 {
139 std::size_t operator()(const FilenameOffsetPair& k) const
140 {
141 return std::hash<std::string>()(k.filename_) ^
142 std::hash<vsi_l_offset>()(k.offset_);
143 }
144 };
145
146 using RegionCacheType =
147 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
148 lru11::NullLock,
149 std::unordered_map<
150 FilenameOffsetPair,
151 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
152 std::shared_ptr<std::string>>>::iterator,
153 FilenameOffsetPairHasher>>;
154
155 RegionCacheType oRegionCache;
156
157 lru11::Cache<std::string, FileProp> oCacheFileProp;
158
159 int nCachedFilesInDirList = 0;
160 lru11::Cache<std::string, CachedDirList> oCacheDirList;
161
162 char** ParseHTMLFileList(const char* pszFilename,
163 int nMaxFiles,
164 char* pszData,
165 bool* pbGotFileList);
166
167protected:
168 CPLMutex *hMutex = nullptr;
169
170 virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
171 virtual char** GetFileList(const char *pszFilename,
172 int nMaxFiles,
173 bool* pbGotFileList);
174
175 void RegisterEmptyDir( const CPLString& osDirname );
176
177 bool AnalyseS3FileList( const CPLString& osBaseURL,
178 const char* pszXML,
179 CPLStringList& osFileList,
180 int nMaxFiles,
181 bool bIgnoreGlacierStorageClass,
182 bool& bIsTruncated );
183
184 void AnalyseSwiftFileList( const CPLString& osBaseURL,
185 const CPLString& osPrefix,
186 const char* pszJson,
187 CPLStringList& osFileList,
188 int nMaxFilesThisQuery,
189 int nMaxFiles,
190 bool& bIsTruncated,
191 CPLString& osNextMarker );
192
193 static const char* GetOptionsStatic();
194
195 static bool IsAllowedFilename( const char* pszFilename );
196
197public:
198 VSICurlFilesystemHandler();
199 ~VSICurlFilesystemHandler() override;
200
201 VSIVirtualHandle *Open( const char *pszFilename,
202 const char *pszAccess,
203 bool bSetError ) override;
204
205 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
206 int nFlags ) override;
207 int Unlink( const char *pszFilename ) override;
208 int Rename( const char *oldpath, const char *newpath ) override;
209 int Mkdir( const char *pszDirname, long nMode ) override;
210 int Rmdir( const char *pszDirname ) override;
211 char **ReadDir( const char *pszDirname ) override
212 { return ReadDirEx(pszDirname, 0); }
213 char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
214
215 int HasOptimizedReadMultiRange( const char* /* pszPath */ )
216 override { return true; }
217
218 const char* GetActualURL(const char* pszFilename) override;
219
220 const char* GetOptions() override;
221
222 char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
223 bool* pbGotFileList );
224 void InvalidateDirContent( const char *pszDirname );
225
226 virtual CPLString GetFSPrefix() { return "/vsicurl/"; }
227 virtual bool AllowCachedDataFor(const char* pszFilename);
228
229 std::shared_ptr<std::string> GetRegion( const char* pszURL,
230 vsi_l_offset nFileOffsetStart );
231
232 void AddRegion( const char* pszURL,
233 vsi_l_offset nFileOffsetStart,
234 size_t nSize,
235 const char *pData );
236
237 bool GetCachedFileProp( const char* pszURL,
238 FileProp& oFileProp );
239 void SetCachedFileProp( const char* pszURL,
240 const FileProp& oFileProp );
241 void InvalidateCachedData( const char* pszURL );
242
243 CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
244
245 virtual void ClearCache();
246 virtual void PartialClearCache(const char* pszFilename);
247
248
249 bool GetCachedDirList( const char* pszURL,
250 CachedDirList& oCachedDirList );
251 void SetCachedDirList( const char* pszURL,
252 const CachedDirList& oCachedDirList );
253 bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
254
255 virtual CPLString GetURLFromFilename( const CPLString& osFilename );
256};
257
258/************************************************************************/
259/* VSICurlHandle */
260/************************************************************************/
261
262class VSICurlHandle : public VSIVirtualHandle
263{
264 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
265
266 protected:
267 VSICurlFilesystemHandler* poFS = nullptr;
268
269 bool m_bCached = true;
270
271 FileProp oFileProp{};
272
273 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
274 char* m_pszURL = nullptr; // e.g "http://example.com/foo"
275
276 char **m_papszHTTPOptions = nullptr;
277
278 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
279 int nBlocksToDownload = 1;
280
281 bool bStopOnInterruptUntilUninstall = false;
282 bool bInterrupted = false;
283 VSICurlReadCbkFunc pfnReadCbk = nullptr;
284 void *pReadCbkUserData = nullptr;
285
286 int m_nMaxRetry = 0;
287 double m_dfRetryDelay = 0.0;
288
289 void DownloadRegionPostProcess( const vsi_l_offset startOffset,
290 const int nBlocks,
291 const char* pBuffer,
292 size_t nSize );
293
294 private:
295
296 vsi_l_offset curOffset = 0;
297
298 bool bEOF = false;
299
300 virtual bool DownloadRegion(vsi_l_offset startOffset, int nBlocks);
301
302 bool m_bUseHead = false;
303
304 int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
305 const vsi_l_offset* panOffsets,
306 const size_t* panSizes );
307 CPLString GetRedirectURLIfValid(bool& bHasExpired);
308
309 protected:
310 virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
311 const struct curl_slist* /* psExistingHeaders */)
312 { return nullptr; }
313 virtual bool AllowAutomaticRedirection() { return true; }
314 virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
315 virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
316 virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
317 virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
318 void SetURL(const char* pszURL);
319
320 public:
321
322 VSICurlHandle( VSICurlFilesystemHandler* poFS,
323 const char* pszFilename,
324 const char* pszURLIn = nullptr );
325 ~VSICurlHandle() override;
326
327 int Seek( vsi_l_offset nOffset, int nWhence ) override;
328 vsi_l_offset Tell() override;
329 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
330 int ReadMultiRange( int nRanges, void ** ppData,
331 const vsi_l_offset* panOffsets,
332 const size_t* panSizes ) override;
333 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
334 int Eof() override;
335 int Flush() override;
336 int Close() override;
337
338 bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
339 vsi_l_offset GetFileSize() { return GetFileSize(false); }
340 virtual vsi_l_offset GetFileSize( bool bSetError );
341 bool Exists( bool bSetError );
342 bool IsDirectory() const { return oFileProp.bIsDirectory; }
343 time_t GetMTime() const { return oFileProp.mTime; }
344
345 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
346 void* pfnUserData,
347 int bStopOnInterruptUntilUninstall );
348 int UninstallReadCbk();
349
350 const char *GetURL() const { return m_pszURL; }
351};
352
353/************************************************************************/
354/* IVSIS3LikeFSHandler */
355/************************************************************************/
356
357class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
358{
359 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
360
361 protected:
362 char** GetFileList( const char *pszFilename,
363 int nMaxFiles,
364 bool* pbGotFileList ) override;
365
366 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
367 const char* pszURI, bool bAllowNoObject) = 0;
368
369 IVSIS3LikeFSHandler() = default;
370
371 public:
372 int Unlink( const char *pszFilename ) override;
373 int Mkdir( const char *pszDirname, long nMode ) override;
374 int Rmdir( const char *pszDirname ) override;
375 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
376 int nFlags ) override;
377
378 virtual int DeleteObject( const char *pszFilename );
379
380 virtual const char* GetDebugKey() const = 0;
381
382 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
383 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
384
385 bool Sync( const char* pszSource, const char* pszTarget,
386 const char* const * papszOptions,
387 GDALProgressFunc pProgressFunc,
388 void *pProgressData,
389 char*** ppapszOutputs ) override;
390
391 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
392 const char* const *papszOptions) override;
393};
394
395/************************************************************************/
396/* IVSIS3LikeHandle */
397/************************************************************************/
398
399class IVSIS3LikeHandle: public VSICurlHandle
400{
401 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
402
403 protected:
404 bool UseLimitRangeGetInsteadOfHead() override { return true; }
405 bool IsDirectoryFromExists( const char* pszVerb,
406 int response_code ) override
407 {
408 // A bit dirty, but on S3, a GET on a existing directory returns a 416
409 return response_code == 416 && EQUAL(pszVerb, "GET") &&
410 CPLString(m_pszURL).back() == '/';
411 }
412 void ProcessGetFileSizeResult( const char* pszContent ) override
413 {
414 oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
415 }
416
417 public:
418 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
419 const char* pszFilename,
420 const char* pszURLIn = nullptr ) :
421 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
422 ~IVSIS3LikeHandle() override {}
423};
424
425/************************************************************************/
426/* VSIS3WriteHandle */
427/************************************************************************/
428
429class VSIS3WriteHandle final : public VSIVirtualHandle
430{
431 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
432
433 IVSIS3LikeFSHandler *m_poFS = nullptr;
434 CPLString m_osFilename{};
435 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
436 bool m_bUseChunked = false;
437
438 vsi_l_offset m_nCurOffset = 0;
439 int m_nBufferOff = 0;
440 int m_nBufferSize = 0;
441 int m_nBufferOffReadCallback = 0;
442 bool m_bClosed = false;
443 GByte *m_pabyBuffer = nullptr;
444 CPLString m_osUploadID{};
445 int m_nPartNumber = 0;
446 std::vector<CPLString> m_aosEtags{};
447 CPLString m_osXML{};
448 int m_nOffsetInXML = 0;
449 bool m_bError = false;
450
451 CURLM *m_hCurlMulti = nullptr;
452 CURL *m_hCurl = nullptr;
453 const void *m_pBuffer = nullptr;
454 CPLString m_osCurlErrBuf{};
455 size_t m_nChunkedBufferOff = 0;
456 size_t m_nChunkedBufferSize = 0;
457
458 static size_t ReadCallBackBuffer( char *buffer, size_t size,
459 size_t nitems, void *instream );
460 bool InitiateMultipartUpload();
461 bool UploadPart();
462 static size_t ReadCallBackXML( char *buffer, size_t size,
463 size_t nitems, void *instream );
464 bool CompleteMultipart();
465 bool AbortMultipart();
466 bool DoSinglePartPUT();
467
468 static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
469 size_t nitems, void *instream );
470 size_t WriteChunked( const void *pBuffer,
471 size_t nSize, size_t nMemb );
472 int FinishChunkedTransfer();
473
474 void InvalidateParentDirectory();
475
476 public:
477 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
478 const char* pszFilename,
479 IVSIS3LikeHandleHelper* poS3HandleHelper,
480 bool bUseChunked );
481 ~VSIS3WriteHandle() override;
482
483 int Seek( vsi_l_offset nOffset, int nWhence ) override;
484 vsi_l_offset Tell() override;
485 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
486 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
487 int Eof() override;
488 int Close() override;
489
490 bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
491};
492
493/************************************************************************/
494/* VSIAppendWriteHandle */
495/************************************************************************/
496
497class VSIAppendWriteHandle : public VSIVirtualHandle
498{
499 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
500
501 protected:
502
503 VSICurlFilesystemHandler* m_poFS = nullptr;
504 CPLString m_osFSPrefix{};
505 CPLString m_osFilename{};
506
507 vsi_l_offset m_nCurOffset = 0;
508 int m_nBufferOff = 0;
509 int m_nBufferSize = 0;
510 int m_nBufferOffReadCallback = 0;
511 bool m_bClosed = false;
512 GByte *m_pabyBuffer = nullptr;
513 bool m_bError = false;
514
515 static size_t ReadCallBackBuffer( char *buffer, size_t size,
516 size_t nitems, void *instream );
517 virtual bool Send(bool bIsLastBlock) = 0;
518
519 public:
520 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
521 const char* pszFSPrefix,
522 const char* pszFilename,
523 int nChunkSize );
524 virtual ~VSIAppendWriteHandle();
525
526 int Seek( vsi_l_offset nOffset, int nWhence ) override;
527 vsi_l_offset Tell() override;
528 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
529 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
530 int Eof() override;
531 int Close() override;
532
533 bool IsOK() { return m_pabyBuffer != nullptr; }
534};
535
536int VSICURLGetDownloadChunkSize();
537
538void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
539 VSILFILE *fp,
540 VSICurlReadCbkFunc pfnReadCbk,
541 void *pReadCbkUserData );
542size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
543 size_t nmemb, void *req );
544void MultiPerform(CURLM* hCurlMultiHandle,
545 CURL* hEasyHandle = nullptr);
546void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
547
548} // namespace cpl
549
551
552#endif // HAVE_CURL
553
554#endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:439
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:330
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
cpl_port.h
Core portability definitions for CPL.
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:561
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:989
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
cpl_string.h
Various convenience functions for working with strings and string lists.
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:307
VSIStatBufL
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:192
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
Generated for GDAL by doxygen 1.9.4.