一、读写TXT

    /// <summary>
        /// 读取txt
        /// </summary>
        public static string[] ReadTxt(string path)
        {
            List<string> str = new List<string>();
            StreamReader sr = new StreamReader(path, Encoding.Default);
            String line;
            while ((line = sr.ReadLine()) != null)
            {
                str.Add(line.ToString());
            }
            string[] strArray = str.ToArray();
            sr.Close();
            return strArray;
        }

        /// <summary>
        /// 保存txt
        /// </summary>
        /// <param name="rb"></param>
        public static void SaveTxt(string[] rb)
        {
            string path = SaveDialog("(*.txt)|*.txt");
            FileStream srm = File.Open(path, FileMode.OpenOrCreate, FileAccess.Write);
            srm.Seek(0, SeekOrigin.Begin);
            srm.SetLength(0); //清空txt文件
            srm.Close();
            foreach (string str in rb)
            {
                File.AppendAllText(path, str + "\r\n", Encoding.Default);
            }
        }

二、读取嵌入的txt文本文件

       public StreamReader getResource(string name)
        {
            StreamReader sr;
            Assembly asm = this.GetType().Assembly;
            name = this.GetType().Namespace + "." + name;
            Stream strm = asm.GetManifestResourceStream(name);
            return sr = new StreamReader(strm, Encoding.Default);

            //调用
            ///***读取嵌入的文本资源***/
            //StreamReader sr = getResource("TextFile1.txt");
            //string tmpstr = sr.ReadLine(); //读取第一行
        }

把txt文件放到项目中，修改属性为“嵌入的资源”：

三、实现大数据量txt文本数据快速高效去重

对几千万的TXT文本数据进行去重处理，查找其中重复的数据，并移除。尝试了各种方法，下属方法是目前尝试到最快的方法。以下代码将重复和不重复数据进行分文件存放，提升效率的关键是用到了HashSet。
                TextReader reader = File.OpenText(m_dataFilePath);
                string[] files = new string[2];
                files[0] = ROOT_DIR + "不重复数据.txt";
                files[1] = ROOT_DIR + "重复数据.txt";
                TextWriter writer1 = File.CreateText(files[0]);
                TextWriter writer2 = File.CreateText(files[1]);
                string currentLine;
                int idx = 0;
                HashSet<string> previousLines = new HashSet<string>(new MyEqualityComparer());
                while ((currentLine = reader.ReadLine()) != null)
                {
                    if ((++idx % 10000) == 0)
                        UpdateInfo("正在比对第 " + idx + " 条数据…");
                    currentLine = currentLine.TrimEnd();
                    if (previousLines.Add(currentLine))
                    {
                        writer1.WriteLine(currentLine);
                    }
                    else
                    {
                        if(m_allSave)
                            writer2.WriteLine(currentLine);
                    }
                }
                reader.Close();
                writer1.Close();
                writer2.Close();
                reader.Dispose();
                writer1.Dispose();
                writer2.Dispose();
 

https://blog.csdn.net/weixin_33739523/article/details/85588860