GCC Code Coverage Report
Directory: src/ Exec Total Coverage
File: src/net/download.cpp Lines: 72 272 26.5 %
Date: 2018-11-12 Branches: 31 291 10.7 %

Line Branch Exec Source
1
/*
2
 *  The ManaPlus Client
3
 *  Copyright (C) 2009-2010  The Mana Developers
4
 *  Copyright (C) 2011-2018  The ManaPlus Developers
5
 *
6
 *  This file is part of The ManaPlus Client.
7
 *
8
 *  This program is free software; you can redistribute it and/or modify
9
 *  it under the terms of the GNU General Public License as published by
10
 *  the Free Software Foundation; either version 2 of the License, or
11
 *  any later version.
12
 *
13
 *  This program is distributed in the hope that it will be useful,
14
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16
 *  GNU General Public License for more details.
17
 *
18
 *  You should have received a copy of the GNU General Public License
19
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
20
 */
21
22
#include "net/download.h"
23
24
#include "configuration.h"
25
#include "logger.h"
26
#include "settings.h"
27
28
#include "fs/files.h"
29
30
#include "utils/cast.h"
31
#include "utils/sdlhelper.h"
32
33
#include <zlib.h>
34
35
#include <fstream>
36
#include <sstream>
37
38
#include "debug.h"
39
40
const char *DOWNLOAD_ERROR_MESSAGE_THREAD
41
    = "Could not create download thread!";
42
43
extern volatile bool isTerminate;
44
45
enum
46
{
47
    OPTIONS_NONE = 0,
48
    OPTIONS_MEMORY = 1
49
};
50
51
namespace Net
52
{
53
54
1
std::string Download::mUploadResponse;
55
56
2
Download::Download(void *const ptr,
57
                   const std::string &url,
58
                   const DownloadUpdate updateFunction,
59
                   const bool ignoreError,
60
                   const bool isUpload,
61
2
                   const bool isXml) :
62
    mPtr(ptr),
63
    mUrl(url),
64
    mOptions(),
65
    mFileName(),
66
    mUrlQueue(),
67
    mWriteFunction(nullptr),
68
    mAdler(0),
69
    mUpdateFunction(updateFunction),
70
    mThread(nullptr),
71
    mCurl(nullptr),
72
    mHeaders(nullptr),
73
    mFormPost(nullptr),
74
2
    mError(static_cast<char*>(calloc(CURL_ERROR_SIZE + 1, 1))),
75
    mIgnoreError(ignoreError),
76
    mUpload(isUpload),
77
12
    mIsXml(isXml)
78
{
79
2
    if (mError != nullptr)
80
2
        mError[0] = 0;
81
82
2
    mOptions.cancel = 0;
83
2
    mOptions.memoryWrite = 0;
84
2
    mOptions.checkAdler = 1U;
85
2
    if (!mUpload)
86
    {
87
4
        const std::string serverName = settings.serverName;
88
2
        if (!serverName.empty())
89
        {
90
            if (mUrl.find('?') == std::string::npos)
91
                mUrl.append("?host=");
92
            else
93
                mUrl.append("&host=");
94
            mUrl.append(serverName);
95
        }
96
    }
97
4
    mUrlQueue.push(url);
98
2
}
99
100
10
Download::~Download()
101
{
102
2
    if (mFormPost != nullptr)
103
    {
104
        curl_formfree(mFormPost);
105
        mFormPost = nullptr;
106
    }
107
108
2
    if (mHeaders != nullptr)
109
    {
110
1
        curl_slist_free_all(mHeaders);
111
1
        mHeaders = nullptr;
112
    }
113
114
2
    SDL::WaitThread(mThread);
115
2
    mThread = nullptr;
116
2
    free(mError);
117
2
}
118
119
/**
120
 * Calculates the Alder-32 checksum for the given file.
121
 */
122
unsigned long Download::fadler32(FILE *const file)
123
{
124
    if (file == nullptr)
125
        return 0;
126
127
    // Obtain file size
128
    fseek(file, 0, SEEK_END);
129
    const long fileSize = ftell(file);
130
    if (fileSize < 0)
131
    {   // file size error
132
        return 0;
133
    }
134
    rewind(file);
135
136
    // Calculate Adler-32 checksum
137
    char *const buffer = new char[CAST_SIZE(fileSize)];
138
    const uInt read = static_cast<uInt>(fread(buffer, 1, fileSize, file));
139
    unsigned long adler = adler32(0L, nullptr, 0);
140
    adler = adler32(static_cast<uInt>(adler),
141
        reinterpret_cast<Bytef*>(buffer), read);
142
    delete [] buffer;
143
    return adler;
144
}
145
146
unsigned long Download::adlerBuffer(const char *const buffer, int size)
147
{
148
    FUNC_BLOCK("Download::adlerBuffer", 1)
149
    unsigned long adler = adler32(0L, nullptr, 0);
150
    return adler32(static_cast<uInt>(adler),
151
        reinterpret_cast<const Bytef*>(buffer), size);
152
}
153
154
void Download::addHeader(const std::string &header)
155
{
156

2
    mHeaders = curl_slist_append(mHeaders, header.c_str());
157
}
158
159
1
void Download::noCache()
160
{
161
5
    addHeader("pragma: no-cache");
162
5
    addHeader("Cache-Control: no-cache");
163
1
}
164
165
1
void Download::setFile(const std::string &filename, const int64_t adler32)
166
{
167
1
    mOptions.memoryWrite = 0;
168
2
    mFileName = filename;
169
170
1
    if (adler32 > -1)
171
    {
172
        mAdler = static_cast<unsigned long>(adler32);
173
        mOptions.checkAdler = 1U;
174
    }
175
    else
176
    {
177
1
        mOptions.checkAdler = 0;
178
    }
179
1
}
180
181
1
void Download::setWriteFunction(WriteFunction write)
182
{
183
1
    mOptions.memoryWrite = 1U;
184
1
    mWriteFunction = write;
185
1
}
186
187
2
bool Download::start()
188
{
189
2
    if (mUpload)
190
        logger->log("Starting upload: %s", mUrl.c_str());
191
    else
192
4
        logger->log("Starting download: %s", mUrl.c_str());
193
194
2
    mThread = SDL::createThread(&downloadThread, "download", this);
195
2
    if (mThread == nullptr)
196
    {
197
        logger->log1(DOWNLOAD_ERROR_MESSAGE_THREAD);
198
        if (mError != nullptr)
199
            strcpy(mError, DOWNLOAD_ERROR_MESSAGE_THREAD);
200
        mUpdateFunction(mPtr, DownloadStatus::ThreadError, 0, 0);
201
        if (!mIgnoreError)
202
            return false;
203
    }
204
205
    return true;
206
}
207
208
2
void Download::cancel()
209
{
210
4
    logger->log("Canceling download: %s", mUrl.c_str());
211
212
2
    mOptions.cancel = 1U;
213
2
    SDL::WaitThread(mThread);
214
2
    mThread = nullptr;
215
2
}
216
217
const char *Download::getError() const
218
{
219
    return mError;
220
}
221
222
int Download::downloadProgress(void *clientp, double dltotal, double dlnow,
223
                               double ultotal A_UNUSED, double ulnow A_UNUSED)
224
{
225
    Download *const d = reinterpret_cast<Download *>(clientp);
226
227
    if (d == nullptr)
228
        return -5;
229
230
    if (d->mUpload)
231
        return 0;
232
233
    if (d->mOptions.cancel != 0U)
234
    {
235
        return d->mUpdateFunction(d->mPtr, DownloadStatus::Cancelled,
236
                                  CAST_SIZE(dltotal),
237
                                  CAST_SIZE(dlnow));
238
    }
239
240
    return d->mUpdateFunction(d->mPtr, DownloadStatus::Idle,
241
                              CAST_SIZE(dltotal),
242
                              CAST_SIZE(dlnow));
243
}
244
245
2
int Download::downloadThread(void *ptr)
246
{
247
2
    int attempts = 0;
248
2
    bool complete = false;
249
2
    Download *const d = reinterpret_cast<Download*>(ptr);
250
    CURLcode res;
251
252
2
    if (d == nullptr)
253
        return 0;
254
255
2
    std::string outFilename;
256
2
    if (d->mUpload)
257
    {
258
        outFilename = d->mFileName;
259
        prepareForm(&d->mFormPost, outFilename);
260
    }
261
    else
262
    {
263
2
        if (d->mOptions.memoryWrite == 0U)
264
2
            outFilename = d->mFileName + ".part";
265
        else
266
            outFilename.clear();
267
    }
268
269
8
    while (!d->mUrlQueue.empty())
270
    {
271
3
        attempts = 0;
272
3
        complete = false;
273
9
        d->mUrl = d->mUrlQueue.front();
274
6
        d->mUrlQueue.pop();
275
276
6
        logger->log_r("selected url: %s", d->mUrl.c_str());
277
9
        while (attempts < 3 &&
278
6
               !complete &&
279

5
               (d->mOptions.cancel == 0U) &&
280
2
               isTerminate == false)
281
        {
282
            d->mUpdateFunction(d->mPtr, DownloadStatus::Starting, 0, 0);
283
284
            if ((d->mOptions.cancel != 0U) || isTerminate == true)
285
            {
286
                return 0;
287
            }
288
            d->mCurl = curl_easy_init();
289
290
            if (d->mCurl != nullptr &&
291
                d->mOptions.cancel == 0U &&
292
                isTerminate == false)
293
            {
294
                FILE *file = nullptr;
295
296
                if (d->mUpload)
297
                {
298
                    logger->log_r("Uploading: %s", d->mUrl.c_str());
299
                    curl_easy_setopt(d->mCurl, CURLOPT_URL, d->mUrl.c_str());
300
                    curl_easy_setopt(d->mCurl, CURLOPT_HTTPPOST, d->mFormPost);
301
                    curl_easy_setopt(d->mCurl, CURLOPT_WRITEFUNCTION,
302
                        &Download::writeFunction);
303
                    mUploadResponse.clear();
304
                }
305
                else
306
                {
307
                    logger->log_r("Downloading: %s", d->mUrl.c_str());
308
                    curl_easy_setopt(d->mCurl, CURLOPT_FOLLOWLOCATION, 1);
309
                    curl_easy_setopt(d->mCurl, CURLOPT_HTTPHEADER,
310
                        d->mHeaders);
311
                    if (d->mOptions.memoryWrite != 0U)
312
                    {
313
                        curl_easy_setopt(d->mCurl, CURLOPT_FAILONERROR, 1);
314
                        curl_easy_setopt(d->mCurl, CURLOPT_WRITEFUNCTION,
315
                                         d->mWriteFunction);
316
                        curl_easy_setopt(d->mCurl, CURLOPT_WRITEDATA, d->mPtr);
317
                    }
318
                    else
319
                    {
320
                        file = fopen(outFilename.c_str(), "w+b");
321
                        if (file != nullptr)
322
                        {
323
                            curl_easy_setopt(d->mCurl, CURLOPT_WRITEDATA,
324
                                file);
325
                        }
326
                    }
327
                    curl_easy_setopt(d->mCurl,
328
                        CURLOPT_USERAGENT,
329
                        settings.userAgent.c_str());
330
331
                    curl_easy_setopt(d->mCurl, CURLOPT_ERRORBUFFER, d->mError);
332
                    curl_easy_setopt(d->mCurl, CURLOPT_URL, d->mUrl.c_str());
333
                    curl_easy_setopt(d->mCurl, CURLOPT_NOPROGRESS, 0);
334
                    curl_easy_setopt(d->mCurl, CURLOPT_PROGRESSFUNCTION,
335
                        &downloadProgress);
336
                    curl_easy_setopt(d->mCurl, CURLOPT_PROGRESSDATA, ptr);
337
#if LIBCURL_VERSION_NUM >= 0x070a00
338
                    curl_easy_setopt(d->mCurl, CURLOPT_NOSIGNAL, 1);
339
#endif  // LIBCURL_VERSION_NUM >= 0x070a00
340
                    curl_easy_setopt(d->mCurl, CURLOPT_CONNECTTIMEOUT, 30);
341
                    curl_easy_setopt(d->mCurl, CURLOPT_TIMEOUT, 1800);
342
                    addHeaders(d->mCurl);
343
                    addProxy(d->mCurl);
344
                    secureCurl(d->mCurl);
345
                }
346
                addCommonFlags(d->mCurl);
347
348
                if ((res = curl_easy_perform(d->mCurl)) != 0 &&
349
                    (d->mOptions.cancel == 0U) &&
350
                    isTerminate == false)
351
                {
352
                    PRAGMA45(GCC diagnostic push)
353
                    PRAGMA45(GCC diagnostic ignored "-Wswitch-enum")
354
                    switch (res)
355
                    {
356
                        case CURLE_ABORTED_BY_CALLBACK:
357
                            d->mOptions.cancel = 1U;
358
                            break;
359
                        case CURLE_COULDNT_CONNECT:
360
                        default:
361
                            break;
362
                    }
363
                    PRAGMA45(GCC diagnostic pop)
364
365
                    if (res != 0U)
366
                    {
367
                        if (d->mError != nullptr)
368
                        {
369
                            logger->log_r("curl error %d: %s host: %s",
370
                                res, d->mError, d->mUrl.c_str());
371
                        }
372
                        attempts++;
373
                        continue;
374
                    }
375
376
                    if ((d->mOptions.cancel != 0U) || isTerminate == true)
377
                        break;
378
379
//                    d->mUpdateFunction(d->mPtr, DownloadStatus::Error, 0, 0);
380
381
                    if (file != nullptr)
382
                    {
383
                        fclose(file);
384
                        file = nullptr;
385
                    }
386
                    if (!d->mUpload && (d->mOptions.memoryWrite == 0U))
387
                        ::remove(outFilename.c_str());
388
                    attempts++;
389
                    continue;
390
                }
391
392
                curl_easy_cleanup(d->mCurl);
393
                d->mCurl = nullptr;
394
395
                if (d->mUpload)
396
                {
397
                    if (file != nullptr)
398
                    {
399
                        fclose(file);
400
                        file = nullptr;
401
                    }
402
                    // need check first if we read data from server
403
                    complete = true;
404
                }
405
                else
406
                {
407
                    if (d->mOptions.memoryWrite == 0U)
408
                    {
409
                        // Don't check resources.xml checksum
410
                        if (d->mOptions.checkAdler != 0U)
411
                        {
412
                            const unsigned long adler = fadler32(file);
413
414
                            if (d->mAdler != adler)
415
                            {
416
                                if (file != nullptr)
417
                                {
418
                                    fclose(file);
419
                                    file = nullptr;
420
                                }
421
422
                                // Remove the corrupted file
423
                                ::remove(d->mFileName.c_str());
424
                                logger->log_r("Checksum for file %s failed:"
425
                                    " (%lx/%lx)",
426
                                    d->mFileName.c_str(),
427
                                    adler, d->mAdler);
428
                                attempts++;
429
                                continue;
430
                            }
431
                        }
432
433
                        if (file != nullptr)
434
                        {
435
                            fclose(file);
436
                            file = nullptr;
437
                        }
438
439
                        // Any existing file with this name is deleted first,
440
                        // otherwise the rename will fail on Windows.
441
                        if ((d->mOptions.cancel == 0U) && isTerminate == false)
442
                        {
443
                            if (d->mIsXml)
444
                            {
445
                                if (!XML::Document::validateXml(outFilename))
446
                                {
447
                                    logger->log_r("Xml file validation error");
448
                                    attempts++;
449
                                    continue;
450
                                }
451
                            }
452
453
                            ::remove(d->mFileName.c_str());
454
                            Files::renameFile(outFilename, d->mFileName);
455
456
                            // Check if we can open it and no errors were
457
                            // encountered during renaming
458
                            file = fopen(d->mFileName.c_str(), "rb");
459
                            if (file != nullptr)
460
                            {
461
                                fclose(file);
462
                                file = nullptr;
463
                                complete = true;
464
                            }
465
                        }
466
                    }
467
                    else
468
                    {
469
                        // It's stored in memory, we're done
470
                        complete = true;
471
                    }
472
                }
473
            }
474
475
            if (d->mCurl != nullptr)
476
            {
477
                curl_easy_cleanup(d->mCurl);
478
                d->mCurl = nullptr;
479
            }
480
481
            if ((d->mOptions.cancel != 0U) || isTerminate == true)
482
            {
483
                return 0;
484
            }
485
            attempts++;
486
        }
487
488

3
        if ((complete && attempts < 3) || (d->mOptions.cancel != 0U))
489
            break;
490
    }
491
492

2
    if ((d->mOptions.cancel != 0U) || isTerminate == true)
493
    {
494
        // Nothing to do...
495
    }
496
    else if (!complete || attempts >= 3)
497
    {
498
        d->mUpdateFunction(d->mPtr, DownloadStatus::Error, 0, 0);
499
    }
500
    else
501
    {
502
        d->mUpdateFunction(d->mPtr, DownloadStatus::Complete, 0, 0);
503
    }
504
505
    return 0;
506
}
507
508
void Download::addProxy(CURL *const curl)
509
{
510
    const int mode = config.getIntValue("downloadProxyType");
511
    if (mode == 0)
512
        return;
513
514
    if (mode > 1)
515
    {
516
        curl_easy_setopt(curl, CURLOPT_PROXY,
517
            config.getStringValue("downloadProxy").c_str());
518
    }
519
520
    switch (mode)
521
    {
522
        case 1:  // direct connection
523
        default:
524
            curl_easy_setopt(curl, CURLOPT_PROXY, "");
525
            break;
526
        case 2:  // HTTP
527
#if LIBCURL_VERSION_NUM >= 0x070300
528
            curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL,
529
                config.getIntValue("downloadProxyTunnel"));
530
#endif  // LIBCURL_VERSION_NUM >= 0x070300
531
            break;
532
        case 3:  // HTTP 1.0
533
#if LIBCURL_VERSION_NUM >= 0x071304
534
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP_1_0);
535
#endif  // LIBCURL_VERSION_NUM >= 0x071304
536
#if LIBCURL_VERSION_NUM >= 0x070300
537
            curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL,
538
                config.getIntValue("downloadProxyTunnel"));
539
#endif  // LIBCURL_VERSION_NUM >= 0x070300
540
            break;
541
        case 4:  // SOCKS4
542
#if LIBCURL_VERSION_NUM >= 0x070a00
543
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
544
#endif  // LIBCURL_VERSION_NUM >= 0x070a00
545
            break;
546
        case 5:  // SOCKS4A
547
#if LIBCURL_VERSION_NUM >= 0x071200
548
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
549
#elif LIBCURL_VERSION_NUM >= 0x071000
550
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
551
#endif  // LIBCURL_VERSION_NUM >= 0x071200
552
553
            break;
554
        case 6:  // SOCKS5
555
#if LIBCURL_VERSION_NUM >= 0x071200
556
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
557
#endif  // LIBCURL_VERSION_NUM >= 0x071200
558
559
            break;
560
        case 7:  // SOCKS5 hostname
561
#if LIBCURL_VERSION_NUM >= 0x071200
562
            curl_easy_setopt(curl, CURLOPT_PROXYTYPE,
563
                CURLPROXY_SOCKS5_HOSTNAME);
564
#endif  // LIBCURL_VERSION_NUM >= 0x071200
565
566
            break;
567
    }
568
}
569
570
#if LIBCURL_VERSION_NUM >= 0x070a08
571
void Download::secureCurl(CURL *const curl)
572
#else  // LIBCURL_VERSION_NUM >= 0x070f01
573
void Download::secureCurl(CURL *const curl A_UNUSED)
574
#endif  // LIBCURL_VERSION_NUM >= 0x070f01
575
{
576
#if LIBCURL_VERSION_NUM >= 0x071304
577
    curl_easy_setopt(curl, CURLOPT_PROTOCOLS,
578
        CURLPROTO_HTTP | CURLPROTO_HTTPS);
579
    curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS,
580
        CURLPROTO_HTTP | CURLPROTO_HTTPS);
581
#endif  // LIBCURL_VERSION_NUM >= 0x071304
582
#if LIBCURL_VERSION_NUM >= 0x071500
583
    curl_easy_setopt(curl, CURLOPT_WILDCARDMATCH, 0);
584
#endif  // LIBCURL_VERSION_NUM >= 0x071500
585
#if LIBCURL_VERSION_NUM >= 0x070f01
586
    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3);
587
#endif  // LIBCURL_VERSION_NUM >= 0x070f01
588
#if LIBCURL_VERSION_NUM >= 0x070a08
589
    curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, 536870912);
590
#endif  // LIBCURL_VERSION_NUM >= 0x070a08
591
592
#if LIBCURL_VERSION_NUM >= 0x073100
593
    curl_easy_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L);
594
#endif  // LIBCURL_VERSION_NUM >= 0x073100
595
596
    curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
597
    curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 60L);
598
}
599
600
#if LIBCURL_VERSION_NUM >= 0x071507
601
void Download::addHeaders(CURL *const curl)
602
{
603
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
604
}
605
#else  // LIBCURL_VERSION_NUM >= 0x071507
606
607
void Download::addHeaders(CURL *const curl A_UNUSED)
608
{
609
}
610
#endif  // LIBCURL_VERSION_NUM >= 0x071507
611
612
void Download::addCommonFlags(CURL *const curl)
613
{
614
    curl_easy_setopt(curl, CURLOPT_STDERR, logger->getFile());
615
#if LIBCURL_VERSION_NUM >= 0x072D00
616
    curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "http");
617
#endif  // LIBCURL_VERSION_NUM >= 0x072D00
618
}
619
620
void Download::prepareForm(curl_httppost **form, const std::string &fileName)
621
{
622
    curl_httppost *lastPtr = nullptr;
623
624
    std::ifstream file;
625
    file.open(fileName.c_str(), std::ios::in);
626
    if (!file.is_open())
627
        return;
628
629
    char *line = new char[10001];
630
    std::ostringstream str;
631
    while (file.getline(line, 10000))
632
        str << line << "\n";
633
634
    delete [] line;
635
636
    curl_formadd(form, &lastPtr,
637
        CURLFORM_COPYNAME, "f:1",
638
        CURLFORM_COPYCONTENTS, str.str().c_str(),
639
        CURLFORM_END);
640
}
641
642
size_t Download::writeFunction(void *ptr,
643
                               size_t size,
644
                               size_t nmemb,
645
                               void *stream A_UNUSED)
646
{
647
    const size_t totalMem = size * nmemb;
648
    char *buf = new char[totalMem + 1];
649
    memcpy(buf, ptr, totalMem);
650
    buf[totalMem] = 0;
651
    mUploadResponse.append(buf);
652
    delete [] buf;
653
    return totalMem;
654
}
655
656
2
}  // namespace Net