-
#!/usr/bin/perl
=pod
=head1 NAME
apt-mirror - apt sources mirroring tool
=head1 SYNOPSIS
apt-mirror [configfile]
=head1 DESCRIPTION
A small and efficient tool that lets
you mirror a part of or
the whole
Debian GNU/Linux distribution or any other apt
sources.
Main features:
* It uses a config similar
to APT's F<>
* It's fully
pool compliant
* It
supports multithreaded downloading
* It supports multiple architectures at
the same time
* It can
automatically remove unneeded files
* It works well on an overloaded
Internet connection
* It
never produces an inconsistent mirror including
while mirroring
* It works
on all POSIX compliant systems with Perl and wget
=head1 COMMENTS
apt-mirror uses F as a
configuration file.
By default it is
tuned to official Debian or Ubuntu mirrors. Change
it for your needs.
After you setup the configuration file
you may run as root:
#
su - apt-mirror -c apt-mirror
Or uncomment the line in
F to enable daily mirror
updates.
=head1 FILES
F
Main configuration file
F
Cron configuration template
F
Mirror places here
F
Place for temporarily downloaded
indexes
F
Log files placed here. URLs
and MD5 checksums also here.
=head1 CONFIGURA
TION
EXAMPLES
The configuration
supports many options, the file is well commented
explaining each
option.
Here
are some sample mirror configuration lines showing
the various supported ways:
Normal:
deb /debian stable
main contrib non-free
Arch
Specific: (many other architectures are supported)
deb-powerpc /debian stable main contrib
non-free
HTTP and FTP Auth
or non-standard port:
deb
http://user:pass@:8080/debian stable main contrib
non-free
HTTPS with sending
Basic HTTP authentication information (plaintext
username and password)
for all
requests:
(this
was
default
behaviour
of
Wget
1.10.2
and
prior
and
is
needed
for
some
servers
with
new
version of Wget)
set auth_no_challenge 1
deb
https://user:pass@:443/debian stable main contrib
non-free
HTTPS without
checking certificate:
set
no_check_certificate 1
deb :443/debian
stable main contrib non-free
Source Mirroring:
deb-src
/debian stable main contrib non-free
=head1 AUTHORS
Dmitry N. Hramtsov E
Brandon Holtsclaw E
=cut
### =pod
和
=cut
之间的内容是
apt-
mirror
的帮助文档，编译器忽略
###
use
warnings;
use strict;
use Fi
le::Copy;#
提供
copy
和
move
函数
use File::Compare;#Compare files or
filehandles
use File::Path;#
具有类似
shell
下的
mkdir<
/p>
的功能
use
File::Basename;#
将文件路径解析为目录、
文件名和后缀
use Fcntl
qw(:flock);#
翻译的
c
语
言
fcntl.h
模块
my $$config_file;
#
默认的配置信息设置在一个关联数
组中
my %config_variables = (
=> 20,
=>
'/var/spool/apt-mirror',#
基目录
下载目录
=> '
$$base_path/skel',#
临时索引下载文件目录，
也就是存放软件仓库的
dists
目录下文件
=> '$$base_path/var',#
配置日志，下载线程
URLS
，
MD5
存放
脚本位置
=>
1,
=> 0,
=> 0,
=> '100m',
=>
1,
=> 0,
=>
0,
=> '$$var_path/',
=> 'off',
=> '',
=>
'',
=> ''
);
my @config_binaries = ();
#<
/p>
数组存放
deb
包解析的分词
my @config_sources
= ();
#
数组存放
deb-
src
解析的分词
my @index_urls;
#
存放
dists
目录下的索引
url
my @childrens
= ();
my %skipclean
= ();
my %clean_directory =
();
#######################
##################################################
######
#######
## Setting up
$$config_file variable
#
设置默认的
配置文件
/etc/apt/,
并且检测其是否有效，是否是普
通文件
$$config_file =
#
Default value
if ( $$_ = shift )
{
die(
$$config_file =
$$_;
}
chomp
$$config_variables{
#########
##################################################
####################
#######
## Common subroutines
#
公共的子程序
sub round_number
{
my $$n = shift;
#
取参数第一个值
my
$$minus = $$n < 0 ? '-' : '';
#
条件判断
然后赋值
- or ''
$$n = abs($$n);
#
取绝对值
$$n
= int( ( $$n + .05 ) * 10 ) / 10;
#
对
n
处理
$$n .= '.0' unless $$n =~ /./;
#
$$n .= '0' if
substr( $$n, ( length($$n) - 1 ), 1 ) eq '.'; #
获取改变
n
字符串
chop $$n if $$n =~ /.dd0$$/;
return
}
###
根据参数将字节转换位
KB<
/p>
、
MB
、
GB
sub format_bytes
{
my $$bytes
= shift;
my $$bytes_out = '0';
my
$$size_name = 'bytes';
my $$KiB
= 1024;
my $$MiB
= 1024 * 1024;
my $$GiB
= 1024 * 1024 *
1024;
if ( $$bytes >=
$$KiB )
{
$$bytes_out =
$$bytes / $$KiB;
$$size_name = 'KiB';
if ( $$bytes >= $$MiB )
{
$$bytes_out = $$bytes / $$MiB;
$$size_name = 'MiB';
if
( $$bytes >= $$GiB )
{
$$bytes_out = $$bytes / $$GiB;
$$size_name = 'GiB';
}
}
$$bytes_out =
round_number($$bytes_out);
}
else
{
$$bytes_out = $$bytes;
$$size_name = 'bytes';
}
return
}
sub get_variable
{
#
从关联数组中获取
key
值对应的
value
值
my $$value =
$$config_variables{ shift @_ };
my
$$count = 16;
while ( $$value =~
s/$$(w+)/$$config_variables{$$1}/xg )
{
die(
}
return $$value;
}
sub
lock_aptmirror
{
#
打开文件
open( LOCK_FILE, '>',
get_variable(
my $$lock = flock(
LOCK_FILE, LOCK_EX | LOCK_NB );
#
给打开的文件句柄上锁
if
( !$$lock )
{
die(
}
}
sub
unlock_aptmirror
{
close(LOCK_FILE);
unlink( get_variable(
}
##
下载文件子程序
sub download_urls
{
my $$stage = shift;
my
@urls;
my $$i = 0;
my
$$pid;
my $$nthreads = get_variable
(
#
获取设置好的线程数
my @args
= ();
local $$| = 1;
@urls = @_;
#
获取参数中传入的
URL
$$nthreads = @urls if @urls < $$nthreads;
#
如果
urls
的数目比设置的线程数小，则调小
线程数为
urls
数
#####
检验一些配置项的信息
if ( get_variable(
{ push( @args,
if ( get_variable(
if
( get_variable(
{
push( @args,
if
(
length(
get_variable(
)
&&
(
get_variable(
eq
'yes'
||
get_variable(
{
if
(
length(
get_variable(
)
)
{
push(
@args,
use_proxy=yes
);
push( @args,
if
(
length(
get_variable(
)
)
{
push(
@args,
proxy_user=
.
get_variable(
if ( length(
get_variable(
get_variable(
}
print
while ( scalar @urls )
{
my @part = splice( @urls, 0, int( @urls
/ $$nthreads ) );
#
把
@urls
中的
url
按线程数<
/p>
平均分开，从
@urls
数组的后面开始
移出，并赋值给
@part
open
URLS,
get_variabl
e(
or
die(
can't
write
to intermediate file ($$stage-urls.$$i)
foreach (@part) { print URLS
close URLS or die(
###
fork
子进程
$$pid = fork();
die(
if ( $$pid == 0
)
{
#########
在子进程中用
w
get
下载
exec 'wget', '--no-cache', '--limit-
rate=' . get_variable(
'-l',
'inf',
'-o',
get_variable(
.
'-i',
get_variable(
.
# shouldn't
reach this unless exec fails
die(
}
push @childrens, $$pid;
$$i++;
$$nthreads--;
}
print
##
打印时间和进程
while ( scalar @childrens )
{
my $$dead = wait();
@childrens = grep { $$_ != $$dead }
@childrens;
print
}
print
}
## Parse config
#
解析配置
open CONFIG,
以只读
方式打开配置文件
while (
#
按行读取打开的配置文件
{
next if /^s*#/; #
跳过以
开头的行，
#
之前可以有多个空格。即跳过注释
next unless /S/;
my
@config_line = split;
#
分割该行信
息存入数组
@config_line
my
$$config_line = shift @config_line;
#
去
数
组
中
第
一
个
元
素
赋
值
给
简
单
变
量
$$config_line
if (
$$config_line eq
#
如果解析出该行文件的第
一个元素是
set
{
$$config_variables{ $$config_line[0] } =
$$config_line[1];
#
则
修
改
全
局
的
关
联
数
组<
/p>
%config_variables
的
value
next;
}
if ( $$config_line eq
#
如果解析出该行文件的第一个元素是
deb
{
push @config_binaries, [ ge
t_variable(
#
将该行文件
填入到
config_binaries
数组，并且数组的第一
个元素是解析出的当前系统的机器架构
next;
}
if ( $$config_line eq
#
如果解析出该行文件的第一个元素是
deb-src
，
将该行
文件填入到
config_s
ources
数组
{
push @config_sources, [@config_line];
next;
}
if ( $$config_line =~ /deb-([w-]+)/ )
#
解析出其他
{
push @config_binaries, [
$$1, @config_line ];
next;
}
if (
$$config_line eq
#
解析出其他
{
$$config_line[0] =~
s[^(w+)://][];
$$config_line[0] =~ s[/$$][];
$$config_line[0] =~ s[~][%7E]g if
get_variable(
$$skipclean{ $$config_line[0]
} = 1;
next;
}
if ( $$config_line eq
#
解析出是
clean
开头的文件行
{
$$config_line[0] =~ s[^(w+)://][];
$$config_line[0] =~ s[/$$][];
$$config_line[0] =~ s[~][%7E]g if
get_variable(
$$clean_directory{
$$config_line[0] } = 1;
##
填入
全局的
clean_directory
关联数组
中
next;
}
die(
}
close
CONFIG;
#
关闭打开的文件
die(
##################################################
#############################
#######
## Create the 3 needed directories if
they don't exist yet
#
用
makedir
创建三个必要的目录，如果不存在的话
my
@needed_directories
=
(
get_variable(
get_variable(
get_variable(
foreach my $$needed_directory
(@needed_directories)
{
unless ( -d $$needed_directory )
#-d
判断是否为目录
{
mkdir($$needed_directory) or
die(
}
}
#
#######################################
########################################
########
lock_aptmirror();
#
打开
bash_path/var,
并上锁
#
##################################################
############################
#######
## Skel download
#
下载软件镜像仓库站点的
skel
目录下的索引文件
my
%urls_to_download = ();
#urls_to_download
关联数组
my ( $$url,
$$arch );
#
移除双斜杆
sub remove_double_slashes
{
local $$_ = shift;
while (s[/./][/]g)
{ }
while (s[(?
{ }
while (s[(?
s/~/%7E/g if get_variable(
return $$_;
}
sub add_url_to_download
{
my $$url = remove_double_slashes(shift);
$$urls_to_download{$$url} = shift;
}
###
对源码包的索引地址进行解析存储
foreach (@config_sources)
#<
/p>
将
config_sources
数组<
/p>
/
列表作为参数传入，
{
my ( $$uri, $$distribution,
@components ) = @{$$_};
#config_sources<
/p>
中保存的是
deb-src
的
url
，版本号，属性分类
#
例如：
/ubuntu/
trusty
main
restricted universe
multiverse
if
(@components)
#
对
u
rl
、
distribution
、<
/p>
components
进行解析，
组成能
在
ftp/http
服务器
识别的
url
{
$$url = $$uri .
#
字符串连接操作
add_url_to_download( $$url .
#
将
InRelease
、
Release
、
文件
添加到下载项
add_url_to_download( $$url .
add_url_to_download( $$url .
foreach (@components)
{
add_url_to_download( $$url . $$_ .
#
将
sou
rce
目录下的文
件添加到下载项
add_url_to_download( $$url . $$_ .
add_url_to_download( $$url . $$_ .
add_url_to_download( $$url . $$_ .
}
}
else
{
#
否则，按照软件仓库的其他目录
规则下载
add_url_to_download( $$uri .
add_url_to_download( $$uri .
add_url_to_download( $$uri .
add_url_to_download( $$uri .
add_url_to_download( $$uri .
}
}
###
对
deb
软件包的索引地址解析存储，解析的含义同上面的<
/p>
deb-src
解析
foreach (@config_binaries)
{
my ( $$arch, $$uri, $$distribution,
@components ) = @{$$_};
if
(@components)
{
$$url = $$uri .
add_url_to_download( $$url .
add_url_to_download( $$url .
add_url_to_download( $$url .
if
( get_variable(
{
add_url_to_download( $$url .
add_url_to_download( $$url .
add_url_to_download( $$url .
}
foreach (@components)
{
if ( get_variable(
{
add_url_to_download( $$url . $$_ .
add_url_to_download( $$url . $$_ .
add_url_to_download( $$url . $$_ .
}
add_url_to_download( $$url .
$$_ .
add_url_to_download( $$url .
$$_ .
add_url_to_download( $$url .
$$_ .
add_url_to_download( $$url .
$$_ .
add_url_to_download( $$url .
$$_ .
}
}
else
{
add_url_to_download( $$uri .
add_url_to_download( $$uri .
add_url_to_download( $$uri .
add_url_to_download( $$uri .
add_url_to_download( $$uri .
}
}
###
改变工作目录到
bash_path/skel_path
，
下载索引
chdir
get_variable(
@index_urls = sort keys
%urls_to_download;
download_urls(
###
下载
index_url
foreach ( keys
%urls_to_download )
{
s[^(w+)://][];
s[~][%7E]g if
get_variable(
$$skipclean{$$_} = 1;
$$skipclean{$$_} = 1 if s[.gz$$][];
$$skipclean{$$_} = 1 if s[.bz2$$][];
$$skipclean{$$_} = 1 if s[.xz$$][];
}
##############
##################################################
###############
#######
##
Translation index download
#
Translation index
下载
%urls_to_download = ();
###
对
url
进行处理
sub
sanitise_uri
{
my
$$uri = shift;
$$uri =~ s[^(w+)://][];
$$uri =~ s/^([^@]+)?@?// if $$uri =~ /@/;
$$uri =~ s&:d+/&/&;
# and port information
$$uri =~ s/~/%7E/g if
get_variable(
return $$uri;
}
###
解析通过
< br>i18n/Index
文件，获取
i18n
目录下
translation
文件的文件名，进行
下载
sub
find_translation_files_in_release
{
# Look in the dists/$$DIST/Release file
for the translation files that belong
#
to the given component.
my
$$dist_uri
= shift;
my $$component = shift;
my
( $$release_uri, $$release_path, $$line ) = '';