支持断点续传的多线程下载

0. 目录

1. 参数介绍
2. 断点续传和多线程下载的原理
3. 源码
4. 目录介绍
5. 下载测试（多线程和断点续传）
   5.1 下载
   5.2 文件验证

1. 参数介绍

http请求和响应中，有几个header参数和请求/响应的大小有关：

• Range：这是一个RequestHeader，由客户端向服务器发起请求时携带，用来指定本次请求的文件范围（起始Byte位置和截止Byte位置），格式是 bytes=xx-yy，携带了这个header后，服务器将返回指定片段的文件内容，而非整个文件。
• Content-Length：这个一个ResponseHeader，在http响应中携带，用来表明本次返回的响应长度；

• Content-Range：这是一个ResponseHeader，在http响应中携带，用来表明本次返回的文件片段起止位置，以及整个文件的大小，只有请求Header里携带了Range，并且服务器支持的情况下，才会如此响应。

2. 断点续传和多线程下载的原理

通过Range请求，可以直接跳过已下载的数据，从断开的位置恢复下载，这样就实现了断点续传功能；如果服务器支持针对文件片段进行请求，在获取到资源Content-Length之后，就可以使用多线程同时请求多个片段，实现分片下载。

3. 源码

using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace downloader
{
    enum DownloadState
    {
        Init,
        Started,
        Finished
    }
    class Program
    {
        public const string PathBase = @"C:\Lab\download";
        public const string PathCaches = PathBase + @"\Caches";
        //下载的块大小：4MB
        public const int BlockSize = 1 << 22;
        //下载线程数，固定值
        public const int TaskThreads = 8;
        static void Main(string[] args)
        {
            if (!Directory.Exists(PathCaches)) Directory.CreateDirectory(PathCaches);
            List<TaskDownload> tasks = LoadTasks();
            if (tasks.Count() == 0)
            {
                Dictionary<string, string> urls = new Dictionary<string, string>{
                    {"微信", "https://dl.softmgr.qq.com/original/im/WeChatSetup_2.9.0.123.exe" },
                    {"迅雷", "https://mythk.net/soft/ThunderSpeed1.0.34.360.exe" },
                    {"百度网盘", "http://wppkg.baidupcs.com/issue/netdisk/yunguanjia/BaiduNetdisk_6.9.7.4.exe" }
                };
                foreach (var kv in urls) tasks.Add(new TaskDownload(kv.Key, kv.Value));
                Console.Title = $"创建新任务：{urls.Count}";
            }
            else
            {
                tasks.ForEach(t => t.Run(t.Segments.Count > 0 ? TaskThreads : 1));
                Console.Title = $"加载历史任务：{tasks.Count}";
            }
            while (true)
            {
                Console.Clear();
                Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss>") +
                    "\r\n" +
                    string.Join(
                        "\r\n\r\n",
                        tasks.Select(t => $"{t.Name}\t{t.ID} ({t.Segments.Sum(s => s.Transed) >> 10} KB of {t.Length >> 10} KB)\r\n{t.Message}")) +
                   "\r\n");
                if (tasks.Count(t => !t.Finished) == 0) break;
                Thread.Sleep(2000);
            }
            Console.WriteLine("完成。");
            Console.ReadLine();
        }
        /// <summary>
        /// 检查临时目录，如果有未完成的任务，加载到任务列表
        /// </summary>
        /// <returns></returns>
        static List<TaskDownload> LoadTasks()
        {//TaskSaveObject
            List<TaskDownload> tasks = new List<TaskDownload>();
            //遍历Cache目录，读取每个任务的config文件，根据文件内容恢复下载任务
            foreach (string folder in Directory.GetDirectories(PathCaches))
            {
                TaskSaveObject taskConfig =
                    JsonConvert.DeserializeObject<TaskSaveObject>(File.ReadAllText(folder + @"\config"));
                TaskDownload task = new TaskDownload(taskConfig.Name, taskConfig.Url, false)
                {
                    ID = taskConfig.ID,
                    FileName = taskConfig.FileName,
                    Finished = false,
                    Length = taskConfig.Length,
                    Segments = new List<TaskSegment>()
                };
                foreach (SegmentSaveObject obj in taskConfig.Segments)
                {
                    TaskSegment seg =
                        new TaskSegment(task, obj.From, obj.To, obj.SegID)
                        {
                            State = DownloadState.Init,
                            Transed = 0,
                            Message = "++++++++++"
                        };
                    FileInfo fInfo = new FileInfo($@"{PathCaches}\{seg.Task.ID}\{seg.SegID}");

                    if (fInfo.Exists)
                    {
                        //如果片段存在，且等于块大小，说明本片段下载完成，打上Finished标记
                        //如果未完成，删除临时文件重新下载
                        if (fInfo.Length == BlockSize)
                        {
                            seg.State = DownloadState.Finished;
                            seg.Transed = BlockSize;
                            seg.Message = "##########";
                        }
                        else
                        {
                            fInfo.Delete();
                        }
                    }
                    task.Segments.Add(seg);
                }
                tasks.Add(task);
            }
            return tasks;
        }
        public static string MD5(string plain) =>
            BitConverter.ToString(
                ((HashAlgorithm)CryptoConfig.CreateFromName("MD5")).
                ComputeHash(Encoding.UTF8.GetBytes(plain))
                ).Replace("-", "").Substring(8, 16);
    }
    class TaskDownload
    {
        //任务ID，唯一标识
        public string ID { get; set; }
        //分片任务列表
        public List<TaskSegment> Segments =
            new List<TaskSegment>();
        public string Url { get; set; }
        //保存的文件名
        public string FileName { get; set; }
        //任务名，用于界面显示，无实际用途
        public string Name { get; set; }
        //是否完成
        public bool Finished = false;

        //文件总长度 Bytes
        public long Length { get; set; }

        //已下载长度 Bytes
        public long Transed = 0;

        //进度提示信息
        public string Message =>
            string.Join("", Segments.Select(s => s.Message));
        public string DataCache => $@"{Program.PathBase}\Caches\{ID}\";
        public TaskDownload(string name, string url, bool create = true)
        {
            Name = name;
            Url = url;
            if (create)
            {
                string range = GetRange();
                ID = Program.MD5(url);
                FileName = Regex.Match(url, "(?<=/).*?$", RegexOptions.RightToLeft).Value;
                //创建目录
                if (Directory.Exists(DataCache)) Directory.Delete(DataCache, true);
                Directory.CreateDirectory(DataCache);
                if (string.IsNullOrEmpty(range))
                {
                    //如果对Range无响应，使用单线程下载
                    Split(1);
                }
                else
                {
                    Length = Convert.ToInt64(
                        Regex.Match(range, @"\d+$", RegexOptions.RightToLeft).Value);
                    Split(Program.TaskThreads);
                }
            }
        }
        /// <summary>
        /// 下载失败时，用单线程重新下载
        /// </summary>
        public void Restart()
        {
            _ = GetRange();
            Split(1);
        }
        /// <summary>
        /// 下载2B长度内容，获取Content-Length和ETag
        /// </summary>
        /// <returns></returns>
        public string GetRange()
        {
            using WebClient client = new WebClient();
            client.Headers["Range"] = "bytes=0-1";
            client.DownloadData(Url);
            string range = client.ResponseHeaders["Content-Range"];
            //ETag = client.ResponseHeaders["ETag"];
            //if (string.IsNullOrEmpty(ETag)) ETag = client.ResponseHeaders["Last-Modified"];
            return range;
        }
        public void Abort()
        {
            Segments.ForEach(s => s.Abort());
            if (Directory.Exists(DataCache)) Directory.Delete(DataCache, true);
        }
        /// <summary>
        /// 判断分片是否全部完成，如果全部完成，合并临时文件并输出
        /// </summary>
        public void Save()
        {
            if (Segments.Count(s => s.State != DownloadState.Finished) == 0)
            {
                Finished = true;
                using FileStream stream = new FileStream(Path.Combine(Program.PathBase, FileName), FileMode.Append);
                for (int i = 0; i < Segments.Count; i++)
                {
                    byte[] seg = File.ReadAllBytes(Path.Combine(DataCache, $"{i}"));
                    stream.Write(seg, 0, seg.Length);
                }
                Directory.Delete(DataCache, true);
            }
        }
        /// <summary>
        /// 按指定数量对下载任务进行切片
        /// </summary>
        /// <param name="threads"></param>
        void Split(int threads)
        {
            TaskSaveObject saveObj = new TaskSaveObject()
            {
                FileName = FileName,
                ID = ID,
                Length = Length,
                Name = Name,
                Url = Url
            };
            int taskCount = (int)Math.Ceiling(Length * 1.0 / Program.BlockSize);
            for (int i = 0; i < taskCount; i++)
            {
                TaskSegment segment = new TaskSegment(
                        this,
                        i * Program.BlockSize,
                        Math.Min(Length, Program.BlockSize * (i + 1) - 1),
                        i);
                Segments.Add(segment);
                saveObj.Segments.Add(new SegmentSaveObject()
                {
                    From = segment.From,
                    To = segment.To,
                    SegID = segment.SegID
                });
            }

            File.WriteAllText(Path.Combine(DataCache, "config"), JsonConvert.SerializeObject(saveObj));
            Run(threads);
        }
        public void Run(int threads)
        {
            Task.Run(() =>
            {
                while (Segments.Count(s => s.State == DownloadState.Init) > 0 &&
                Segments.Count(s => s.State == DownloadState.Started) < threads)
                {
                    Segments.Where(s => s.State == DownloadState.Init).First().Run();
                    Thread.Sleep(1000);
                }
            });
        }
    }
    class TaskSegment
    {
        public TaskDownload Task { get; set; }
        //分片任务ID
        public int SegID { get; set; }
        //起始字节
        public long From { get; set; }
        //结束字节
        public long To { get; set; }
        //分片任务已下载长度Bytes
        public long Transed { get; set; }
        public string Message { get; set; }
        public DownloadState State { get; set; }
        WebClient client = new WebClient();

        //为了方便解除事件，不使用匿名委托
        DownloadProgressChangedEventHandler handlerDownloading;
        AsyncCompletedEventHandler handlerDownloadCompleted;
        public TaskSegment(TaskDownload task, long from, long to, int id)
        {
            Task = task;
            From = from;
            To = to;
            SegID = id;
            State = DownloadState.Init;
            Message = To > 0 ? $"++++++++++" : "单线程：0KB";
        }
        public void Run()
        {
            State = DownloadState.Started;
            client.Headers["Range"] = $"bytes={From}-" + (To == 0 ? "" : $"{To}");
            client.DownloadFileAsync(new Uri(Task.Url), Path.Combine(Task.DataCache, $"{SegID}"));

            handlerDownloading = (s, e) =>
          {
              Transed = e.BytesReceived;
              int percentage = e.ProgressPercentage / 10;

              Message = To > 0 ?
              new string('#', percentage) + new string('+', 10 - percentage) :
              $"单线程下载：{Transed >> 10}KB";
          };
            client.DownloadProgressChanged += handlerDownloading;

            handlerDownloadCompleted = (s, e) =>
          {
              State = DownloadState.Finished;
              client.DownloadProgressChanged -= handlerDownloading;
              client.DownloadFileCompleted -= handlerDownloadCompleted;
              client.Dispose();
              client = null;
              Task.Save();
              Message = $"##########";
          };
            client.DownloadFileCompleted += handlerDownloadCompleted;
        }
        public void Abort()
        {
            client.DownloadProgressChanged -= handlerDownloading;
            client.DownloadFileCompleted -= handlerDownloadCompleted;
            client.CancelAsync();
            client.Dispose();
            client = null;
            File.Delete(Path.Combine(Task.DataCache, $"{SegID}"));
        }
    }
    /// <summary>
    /// 摘录关键的任务信息和分片信息，用于保存下载任务的状态，供恢复时使用
    /// </summary>
    class TaskSaveObject
    {
        public string ID { get; set; }
        public string Url { get; set; }
        public string FileName { get; set; }
        public string Name { get; set; }
        public long Length { get; set; }
        public List<SegmentSaveObject> Segments =
            new List<SegmentSaveObject>();
    }
    /// <summary>
    /// 摘录关键的任务信息和分片信息，用于保存下载任务的状态，供恢复时使用
    /// </summary>
    class SegmentSaveObject
    {
        public int SegID { get; set; }
        //起始字节位置
        public long From { get; set; }
        //结束字节位置
        public long To { get; set; }
    }
}

4. 目录介绍

5. 下载测试（多线程和断点续传）

5.1 下载

5.2 文件验证