GCC Code Coverage Report
Directory: src/ Exec Total Coverage
File: src/net/download.cpp Lines: 72 272 26.5 %
Date: 2021-03-17 Branches: 31 291 10.7 %

Line Branch Exec Source
1
/*
2
 *  The ManaPlus Client
3
 *  Copyright (C) 2009-2010  The Mana Developers
4
 *  Copyright (C) 2011-2019  The ManaPlus Developers
5
 *  Copyright (C) 2019-2021  Andrei Karas
6
 *
7
 *  This file is part of The ManaPlus Client.
8
 *
9
 *  This program is free software; you can redistribute it and/or modify
10
 *  it under the terms of the GNU General Public License as published by
11
 *  the Free Software Foundation; either version 2 of the License, or
12
 *  any later version.
13
 *
14
 *  This program is distributed in the hope that it will be useful,
15
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 *  GNU General Public License for more details.
18
 *
19
 *  You should have received a copy of the GNU General Public License
20
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
21
 */
22
23
#include "net/download.h"
24
25
#include "configuration.h"
26
#include "logger.h"
27
#include "settings.h"
28
29
#include "fs/files.h"
30
31
#include "utils/cast.h"
32
#include "utils/sdlhelper.h"
33
34
#include <zlib.h>
35
36
#include <fstream>
37
#include <sstream>
38
39
#include "debug.h"
40
41
const char *DOWNLOAD_ERROR_MESSAGE_THREAD
42
    = "Could not create download thread!";
43
44
extern volatile bool isTerminate;
45
46
enum
47
{
48
    OPTIONS_NONE = 0,
49
    OPTIONS_MEMORY = 1
50
};
51
52
namespace Net
53
{
54
55
1
std::string Download::mUploadResponse;
56
57
2
Download::Download(void *const ptr,
58
                   const std::string &url,
59
                   const DownloadUpdate updateFunction,
60
                   const bool ignoreError,
61
                   const bool isUpload,
62
2
                   const bool isXml) :
63
    mPtr(ptr),
64
    mUrl(url),
65
    mOptions(),
66
    mFileName(),
67
    mUrlQueue(),
68
    mWriteFunction(nullptr),
69
    mAdler(0),
70
    mUpdateFunction(updateFunction),
71
    mThread(nullptr),
72
    mCurl(nullptr),
73
    mHeaders(nullptr),
74
    mFormPost(nullptr),
75
2
    mError(static_cast<char*>(calloc(CURL_ERROR_SIZE + 1, 1))),
76
    mIgnoreError(ignoreError),
77
    mUpload(isUpload),
78
12
    mIsXml(isXml)
79
{
80
2
    if (mError != nullptr)
81
2
        mError[0] = 0;
82
83
2
    mOptions.cancel = 0;
84
2
    mOptions.memoryWrite = 0;
85
2
    mOptions.checkAdler = 1U;
86
2
    if (!mUpload)
87
    {
88
4
        const std::string serverName = settings.serverName;
89
2
        if (!serverName.empty())
90
        {
91
            if (mUrl.find('?') == std::string::npos)
92
                mUrl.append("?host=");
93
            else
94
                mUrl.append("&host=");
95
            mUrl.append(serverName);
96
        }
97
    }
98
4
    mUrlQueue.push(url);
99
2
}
100
101
10
Download::~Download()
102
{
103
2
    if (mFormPost != nullptr)
104
    {
105
        curl_formfree(mFormPost);
106
        mFormPost = nullptr;
107
    }
108
109
2
    if (mHeaders != nullptr)
110
    {
111
1
        curl_slist_free_all(mHeaders);
112
1
        mHeaders = nullptr;
113
    }
114
115
2
    SDL::WaitThread(mThread);
116
2
    mThread = nullptr;
117
2
    free(mError);
118
2
}
119
120
/**
121
 * Calculates the Alder-32 checksum for the given file.
122
 */
123
unsigned long Download::fadler32(FILE *const file)
124
{
125
    if (file == nullptr)
126
        return 0;
127
128
    // Obtain file size
129
    fseek(file, 0, SEEK_END);
130
    const long fileSize = ftell(file);
131
    if (fileSize < 0)
132
    {   // file size error
133
        return 0;
134
    }
135
    rewind(file);
136
137
    // Calculate Adler-32 checksum
138
    char *const buffer = new char[CAST_SIZE(fileSize)];
139
    const uInt read = static_cast<uInt>(fread(buffer, 1, fileSize, file));
140
    unsigned long adler = adler32(0L, nullptr, 0);
141
    adler = adler32(static_cast<uInt>(adler),
142
        reinterpret_cast<Bytef*>(buffer), read);
143
    delete [] buffer;
144
    return adler;
145
}
146
147
unsigned long Download::adlerBuffer(const char *const buffer, int size)
148
{
149
    FUNC_BLOCK("Download::adlerBuffer", 1)
150
    unsigned long adler = adler32(0L, nullptr, 0);
151
    return adler32(static_cast<uInt>(adler),
152
        reinterpret_cast<const Bytef*>(buffer), size);
153
}
154
155
void Download::addHeader(const std::string &header)
156
{
157

2
    mHeaders = curl_slist_append(mHeaders, header.c_str());
158
}
159
160
1
void Download::noCache()
161
{
162
5
    addHeader("pragma: no-cache");
163
5
    addHeader("Cache-Control: no-cache");
164
1
}
165
166
1
void Download::setFile(const std::string &filename, const int64_t adler32)
167
{
168
1
    mOptions.memoryWrite = 0;
169
2
    mFileName = filename;
170
171
1
    if (adler32 > -1)
172
    {
173
        mAdler = static_cast<unsigned long>(adler32);
174
        mOptions.checkAdler = 1U;
175
    }
176
    else
177
    {
178
1
        mOptions.checkAdler = 0;
179
    }
180
1
}
181
182
1
void Download::setWriteFunction(WriteFunction write)
183
{
184
1
    mOptions.memoryWrite = 1U;
185
1
    mWriteFunction = write;
186
1
}
187
188
2
bool Download::start()
189
{
190
2
    if (mUpload)
191
        logger->log("Starting upload: %s", mUrl.c_str());
192
    else
193
4
        logger->log("Starting download: %s", mUrl.c_str());
194
195
2
    mThread = SDL::createThread(&downloadThread, "download", this);
196
2
    if (mThread == nullptr)
197
    {
198
        logger->log1(DOWNLOAD_ERROR_MESSAGE_THREAD);
199
        if (mError != nullptr)
200
            strcpy(mError, DOWNLOAD_ERROR_MESSAGE_THREAD);
201
        mUpdateFunction(mPtr, DownloadStatus::ThreadError, 0, 0);
202
        if (!mIgnoreError)
203
            return false;
204
    }
205
206
    return true;
207
}
208
209
2
void Download::cancel()
210
{
211
4
    logger->log("Canceling download: %s", mUrl.c_str());
212
213
2
    mOptions.cancel = 1U;
214
2
    SDL::WaitThread(mThread);
215
2
    mThread = nullptr;
216
2
}
217
218
const char *Download::getError() const
219
{
220
    return mError;
221
}
222
223
int Download::downloadProgress(void *clientp, double dltotal, double dlnow,
224
                               double ultotal A_UNUSED, double ulnow A_UNUSED)
225
{
226
    Download *const d = reinterpret_cast<Download *>(clientp);
227
228
    if (d == nullptr)
229
        return -5;
230
231
    if (d->mUpload)
232
        return 0;
233
234
    if (d->mOptions.cancel != 0U)
235
    {
236
        return d->mUpdateFunction(d->mPtr, DownloadStatus::Cancelled,
237
                                  CAST_SIZE(dltotal),
238
                                  CAST_SIZE(dlnow));
239
    }
240
241
    return d->mUpdateFunction(d->mPtr, DownloadStatus::Idle,
242
                              CAST_SIZE(dltotal),
243
                              CAST_SIZE(dlnow));
244
}
245
246
2
int Download::downloadThread(void *ptr)
247
{
248
2
    int attempts = 0;
249
2
    bool complete = false;
250
2
    Download *const d = reinterpret_cast<Download*>(ptr);
251
    CURLcode res;
252
253
2
    if (d == nullptr)
254
        return 0;
255
256
2
    std::string outFilename;
257
2
    if (d->mUpload)
258
    {
259
        outFilename = d->mFileName;
260
        prepareForm(&d->mFormPost, outFilename);
261
    }
262
    else
263
    {
264
2
        if (d->mOptions.memoryWrite == 0U)
265
2
            outFilename = d->mFileName + ".part";
266
        else
267
            outFilename.clear();
268
    }
269
270
8
    while (!d->mUrlQueue.empty())
271
    {
272
3
        attempts = 0;
273
3
        complete = false;
274
9
        d->mUrl = d->mUrlQueue.front();
275
6
        d->mUrlQueue.pop();
276
277
6
        logger->log_r("selected url: %s", d->mUrl.c_str());
278
9
        while (attempts < 3 &&
279
6
               !complete &&
280

5
               (d->mOptions.cancel == 0U) &&
281
2
               isTerminate == false)
282
        {
283
            d->mUpdateFunction(d->mPtr, DownloadStatus::Starting, 0, 0);
284
285
            if ((d->mOptions.cancel != 0U) || isTerminate == true)
286
            {
287
                return 0;
288
            }
289
            d->mCurl = curl_easy_init();
290
291
            if (d->mCurl != nullptr &&
292
                d->mOptions.cancel == 0U &&
293
                isTerminate == false)
294
            {
295
                FILE *file = nullptr;
296
297
                if (d->mUpload)
298
                {
299
                    logger->log_r("Uploading: %s", d->mUrl.c_str());
300
                    curl_easy_setopt(d->mCurl, CURLOPT_URL, d->mUrl.c_str());
301
                    curl_easy_setopt(d->mCurl, CURLOPT_HTTPPOST, d->mFormPost);
302
                    curl_easy_setopt(d->mCurl, CURLOPT_WRITEFUNCTION,
303
                        &Download::writeFunction);
304
                    mUploadResponse.clear();
305
                }
306
                else
307
                {
308
                    logger->log_r("Downloading: %s", d->mUrl.c_str());
309
                    curl_easy_setopt(d->mCurl, CURLOPT_FOLLOWLOCATION, 1);
310
                    curl_easy_setopt(d->mCurl, CURLOPT_HTTPHEADER,
311
                        d->mHeaders);
312
                    if (d->mOptions.memoryWrite != 0U)
313
                    {
314
                        curl_easy_setopt(d->mCurl, CURLOPT_FAILONERROR, 1);
315
                        curl_easy_setopt(d->mCurl, CURLOPT_WRITEFUNCTION,
316
                                         d->mWriteFunction);
317
                        curl_easy_setopt(d->mCurl, CURLOPT_WRITEDATA, d->mPtr);
318
                    }
319
                    else
320
                    {
321
                        file = fopen(outFilename.c_str(), "w+b");
322
                        if (file != nullptr)
323
                        {
324
                            curl_easy_setopt(d->mCurl, CURLOPT_WRITEDATA,
325
                                file);
326
                        }
327
                    }
328
                    curl_easy_setopt(d->mCurl,
329
                        CURLOPT_USERAGENT,
330
                        settings.userAgent.c_str());
331
332
                    curl_easy_setopt(d->mCurl, CURLOPT_ERRORBUFFER, d->mError);
333
                    curl_easy_setopt(d->mCurl, CURLOPT_URL, d->mUrl.c_str());
334
                    curl_easy_setopt(d->mCurl, CURLOPT_NOPROGRESS, 0);
335
                    curl_easy_setopt(d->mCurl, CURLOPT_PROGRESSFUNCTION,
336
                        &downloadProgress);
337
                    curl_easy_setopt(d->mCurl, CURLOPT_PROGRESSDATA, ptr);
338
#if LIBCURL_VERSION_NUM >= 0x070a00
339
                    curl_easy_setopt(d->mCurl, CURLOPT_NOSIGNAL, 1);
340
#endif  // LIBCURL_VERSION_NUM >= 0x070a00
341
                    curl_easy_setopt(d->mCurl, CURLOPT_CONNECTTIMEOUT, 30);
342
                    curl_easy_setopt(d->mCurl, CURLOPT_TIMEOUT, 1800);
343
                    addHeaders(d->mCurl);
344
                    addProxy(d->mCurl);
345
                    secureCurl(d->mCurl);
346
                }
347
                addCommonFlags(d->mCurl);
348
349
                if ((res = curl_easy_perform(d->mCurl)) != 0 &&
350
                    (d->mOptions.cancel == 0U) &&
351
                    isTerminate == false)
352
                {
353
                    PRAGMA45(GCC diagnostic push)
354
                    PRAGMA45(GCC diagnostic ignored "-Wswitch-enum")
355
                    switch (res)
356
                    {
357
                        case CURLE_ABORTED_BY_CALLBACK:
358
                            d->mOptions.cancel = 1U;
359
                            break;
360
                        case CURLE_COULDNT_CONNECT:
361
                        default:
362
                            break;
363
                    }
364
                    PRAGMA45(GCC diagnostic pop)
365
366
                    if (res != 0U)
367
                    {
368
                        if (d->mError != nullptr)
369
                        {
370
                            logger->log_r("curl error %d: %s host: %s",
371
                                res, d->mError, d->mUrl.c_str());
372
                        }
373
                        attempts++;
374
                        continue;
375
                    }
376
377
                    if ((d->mOptions.cancel != 0U) || isTerminate == true)
378
                        break;
379
380
//                    d->mUpdateFunction(d->mPtr, DownloadStatus::Error, 0, 0);
381
382
                    if (file != nullptr)
383
                    {
384
                        fclose(file);
385
                        file = nullptr;
386
                    }
387
                    if (!d->mUpload && (d->mOptions.memoryWrite == 0U))
388
                        ::remove(outFilename.c_str());
389
                    attempts++;
390
                    continue;
391
                }
392
393
                curl_easy_cleanup(d->mCurl);
394
                d->mCurl = nullptr;
395
396
                if (d->mUpload)
397
                {
398
                    if (file != nullptr)
399
                    {
400
                        fclose(file);
401
                        file = nullptr;
402
                    }
403
                    // need check first if we read data from server
404
                    complete = true;
405
                }
406
                else
407
                {
408
                    if (d->mOptions.memoryWrite == 0U)
409
                    {
410
                        // Don't check resources.xml checksum
411
                        if (d->mOptions.checkAdler != 0U)
412
                        {
413
                            const unsigned long adler = fadler32(file);
414
415
                            if (d->mAdler != adler)
416
                            {
417
                                if (file != nullptr)
418
                                {
419
                                    fclose(file);
420
                                    file = nullptr;
421
                                }
422
423
                                // Remove the corrupted file
424
                                ::remove(d->mFileName.c_str());
425
                                logger->log_r("Checksum for file %s failed:"
426
                                    " (%lx/%lx)",
427
                                    d->mFileName.c_str(),
428
                                    adler, d->mAdler);
429
                                attempts++;
430
                                continue;
431
                            }
432
                        }
433
434
                        if (file != nullptr)
435
                        {
436
                            fclose(file);
437
                            file = nullptr;
438
                        }
439
440
                        // Any existing file with this name is deleted first,
441
                        // otherwise the rename will fail on Windows.
442
                        if ((d->mOptions.cancel == 0U) && isTerminate == false)
443
                        {
444
                            if (d->mIsXml)
445
                            {
446
                                if (!XML::Document::validateXml(outFilename))
447
                                {
448
                                    logger->log_r("Xml file validation error");
449
                                    attempts++;
450
                                    continue;
451
                                }
452
                            }
453
454
                            ::remove(d->mFileName.c_str());
455
                            Files::renameFile(outFilename, d->mFileName);
456
457
                            // Check if we can open it and no errors were
458
                            // encountered during renaming
459
                            file = fopen(d->mFileName.c_str(), "rb");
460
                            if (file != nullptr)
461
                            {
462
                                fclose(file);
463
                                file = nullptr;
464
                                complete = true;
465
                            }
466
                        }
467
                    }
468
                    else
469
                    {
470
                        // It's stored in memory, we're done
471
                        complete = true;
472
                    }
473
                }
474
            }
475
476
            if (d->mCurl != nullptr)
477
            {
478
                curl_easy_cleanup(d->mCurl);
479
                d->mCurl = nullptr;
480
            }
481
482
            if ((d->mOptions.cancel != 0U) || isTerminate == true)
483
            {
484
                return 0;
485
            }
486
            attempts++;
487
        }
488
489

3
        if ((complete && attempts < 3) || (d->mOptions.cancel != 0U))
490
            break;
491
    }
492
493

2
    if ((d->mOptions.cancel != 0U) || isTerminate == true)
494
    {
495
        // Nothing to do...
496
    }
497
    else if (!complete || attempts >= 3)
498
    {
499
        d->mUpdateFunction(d->mPtr, DownloadStatus::Error, 0, 0);
500
    }
501
    else
502
    {
503
        d->mUpdateFunction(d->mPtr, DownloadStatus::Complete, 0, 0);
504
    }
505
506
    return 0;
507
}
508
509
void Download::addProxy(CURL *const curl)
510
{
511
    const int mode = config.getIntValue("downloadProxyType");
512
    if (mode == 0)
513
        return;
514
515
    if (mode > 1)
516
    {
517
        curl_easy_setopt(curl, CURLOPT_PROXY,
518
            config.getStringValue("downloadProxy").c_str());
519
    }
520
521
    switch (mode)
522
    {
523
        case 1:  // direct connection
524
        default:
525
            curl_easy_setopt(curl, CURLOPT_PROXY, "");
526
            break;
527
        case 2:  // HTTP
528
#if LIBCURL_VERSION_NUM >= 0x070300
529
            curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL,
530
                config.getIntValue("downloadProxyTunnel"));
531
#endif  // LIBCURL_VERSION_NUM >= 0x070300
532
            break;
533
        case 3:  // HTTP 1.0
534
#if LIBCURL_VERSION_NUM >= 0x071304
535
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP_1_0);
536
#endif  // LIBCURL_VERSION_NUM >= 0x071304
537
#if LIBCURL_VERSION_NUM >= 0x070300
538
            curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL,
539
                config.getIntValue("downloadProxyTunnel"));
540
#endif  // LIBCURL_VERSION_NUM >= 0x070300
541
            break;
542
        case 4:  // SOCKS4
543
#if LIBCURL_VERSION_NUM >= 0x070a00
544
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
545
#endif  // LIBCURL_VERSION_NUM >= 0x070a00
546
            break;
547
        case 5:  // SOCKS4A
548
#if LIBCURL_VERSION_NUM >= 0x071200
549
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
550
#elif LIBCURL_VERSION_NUM >= 0x071000
551
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
552
#endif  // LIBCURL_VERSION_NUM >= 0x071200
553
554
            break;
555
        case 6:  // SOCKS5
556
#if LIBCURL_VERSION_NUM >= 0x071200
557
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
558
#endif  // LIBCURL_VERSION_NUM >= 0x071200
559
560
            break;
561
        case 7:  // SOCKS5 hostname
562
#if LIBCURL_VERSION_NUM >= 0x071200
563
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE,
564
                CURLPROXY_SOCKS5_HOSTNAME);
565
#endif  // LIBCURL_VERSION_NUM >= 0x071200
566
567
            break;
568
    }
569
}
570
571
#if LIBCURL_VERSION_NUM >= 0x070a08
572
void Download::secureCurl(CURL *const curl)
573
#else  // LIBCURL_VERSION_NUM >= 0x070f01
574
void Download::secureCurl(CURL *const curl A_UNUSED)
575
#endif  // LIBCURL_VERSION_NUM >= 0x070f01
576
{
577
#if LIBCURL_VERSION_NUM >= 0x071304
578
    curl_easy_setopt(curl, CURLOPT_PROTOCOLS,
579
        CURLPROTO_HTTP | CURLPROTO_HTTPS);
580
    curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS,
581
        CURLPROTO_HTTP | CURLPROTO_HTTPS);
582
#endif  // LIBCURL_VERSION_NUM >= 0x071304
583
#if LIBCURL_VERSION_NUM >= 0x071500
584
    curl_easy_setopt(curl, CURLOPT_WILDCARDMATCH, 0);
585
#endif  // LIBCURL_VERSION_NUM >= 0x071500
586
#if LIBCURL_VERSION_NUM >= 0x070f01
587
    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3);
588
#endif  // LIBCURL_VERSION_NUM >= 0x070f01
589
#if LIBCURL_VERSION_NUM >= 0x070a08
590
    curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, 536870912);
591
#endif  // LIBCURL_VERSION_NUM >= 0x070a08
592
593
#if LIBCURL_VERSION_NUM >= 0x073100
594
    curl_easy_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L);
595
#endif  // LIBCURL_VERSION_NUM >= 0x073100
596
597
    curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
598
    curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 60L);
599
}
600
601
#if LIBCURL_VERSION_NUM >= 0x071507
602
void Download::addHeaders(CURL *const curl)
603
{
604
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
605
}
606
#else  // LIBCURL_VERSION_NUM >= 0x071507
607
608
void Download::addHeaders(CURL *const curl A_UNUSED)
609
{
610
}
611
#endif  // LIBCURL_VERSION_NUM >= 0x071507
612
613
void Download::addCommonFlags(CURL *const curl)
614
{
615
    curl_easy_setopt(curl, CURLOPT_STDERR, logger->getFile());
616
#if LIBCURL_VERSION_NUM >= 0x072D00
617
    curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "http");
618
#endif  // LIBCURL_VERSION_NUM >= 0x072D00
619
}
620
621
void Download::prepareForm(curl_httppost **form, const std::string &fileName)
622
{
623
    curl_httppost *lastPtr = nullptr;
624
625
    std::ifstream file;
626
    file.open(fileName.c_str(), std::ios::in);
627
    if (!file.is_open())
628
        return;
629
630
    char *line = new char[10001];
631
    std::ostringstream str;
632
    while (file.getline(line, 10000))
633
        str << line << "\n";
634
635
    delete [] line;
636
637
    curl_formadd(form, &lastPtr,
638
        CURLFORM_COPYNAME, "f:1",
639
        CURLFORM_COPYCONTENTS, str.str().c_str(),
640
        CURLFORM_END);
641
}
642
643
size_t Download::writeFunction(void *ptr,
644
                               size_t size,
645
                               size_t nmemb,
646
                               void *stream A_UNUSED)
647
{
648
    const size_t totalMem = size * nmemb;
649
    char *buf = new char[totalMem + 1];
650
    memcpy(buf, ptr, totalMem);
651
    buf[totalMem] = 0;
652
    mUploadResponse.append(buf);
653
    delete [] buf;
654
    return totalMem;
655
}
656
657
2
}  // namespace Net