1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
|
#!/bin/sh -e
# Iterate over all relevant Osmocom repositories and generate release tarballs for each of the repository tags. The tags
# are queried from the git server without cloning the repositories first, so we can clone them only if we need to build
# a missing tarball. All repositories are deleted afterwards to save space.
#
# Environment variables:
# * KEEP_TEMP: do not delete cloned repositories (use for development)
# * EXISTING_TARBALLS_FROM_JENKINS: set to 1 to get the existing tarballs from
# the last run of the jenkins job
SSH_COMMAND="ssh -o UserKnownHostsFile=/build/known_hosts -p 48"
OSMO_GIT_URL="https://git.osmocom.org"
OSMO_RELEASE_REPOS="
libasn1c
libgtpnl
libosmo-abis
libosmo-netif
libosmo-pfcp
libosmo-sccp
libosmocore
libsmpp34
libusrp
osmo-bsc
osmo-bts
osmo-cbc
osmo-e1d
osmo-gbproxy
osmo-ggsn
osmo-gsm-manuals
osmo-hlr
osmo-hnodeb
osmo-hnbgw
osmo-iuh
osmo-mgw
osmo-msc
osmo-pcap
osmo-pcu
osmo-remsim
osmo-sgsn
osmo-sip-connector
osmo-smlc
osmo-sysmon
osmo-trx
osmo-uecups
osmocom-bb
simtrace2
"
# Print last tags and related commits for an Osmocom git repository, e.g.:
# "ec798b89700dcca5c5b28edf1a1cd16ea311f30a refs/tags/1.0.1"
# $1: Osmocom repository
# $2: amount of commit, tag pairs to print (default: 1, set to "all" to print all)
# $3: string to print when there are no tags (default: empty string)
osmo_git_last_commits_tags() {
# git output:
# ec798b89700dcca5c5b28edf1a1cd16ea311f30a refs/tags/1.0.1
# eab5f594b0a7cf50ad97b039f73beff42cc8312a refs/tags/1.0.1^{}
# ...
# 41e7cf115d4148a9f34fcb863b68b2d5370e335d refs/tags/1.3.1^{}
# 8a9f12dc2f69bf3a4e861cc9a81b71bdc5f13180 refs/tags/3G_2016_09
# ee618ecbedec82dfd240334bc87d0d1c806477b0 refs/tags/debian/0.9.13-0_jrsantos.1
# a3fdd24af099b449c9856422eb099fb45a5595df refs/tags/debian/0.9.13-0_jrsantos.1^{}
# ...
ret="$(git ls-remote --tags "$OSMO_GIT_URL/$1")"
ret="$(echo "$ret" | grep 'refs/tags/[0-9.]*$' || true)"
ret="$(echo "$ret" | sort -V -t/ -k3)"
if [ "$2" != "all" ]; then
ret="$(echo "$ret" | tail -n "$2")"
fi
if [ -n "$ret" ]; then
echo "$ret"
else
echo "$3"
fi
}
cd "$(dirname "$0")"
OUTPUT="/build/_release_tarballs"
TEMP="/build/_temp"
# Print all tags for which no release tarball should be built.
# $1: Osmocom repository
tags_to_ignore() {
case "$1" in
libosmocore)
# configure.ac:144: error: required file 'src/gb/Makefile.in' not found
echo "0.5.0"
echo "0.5.1"
;;
libsmpp34)
# duplicate of 1.12.0
echo "1.12"
;;
osmo-bsc)
# openbsc
echo "1.0.1"
# Requires libosmo-legacy-mgcp
echo "1.1.0"
echo "1.1.1"
echo "1.1.2"
echo "1.2.0"
echo "1.2.1"
echo "1.2.2"
;;
osmo-bts)
# gsm_data_shared.h:464:26: error: field 'power_params' has incomplete type
echo "0.2.0"
echo "0.3.0"
;;
osmo-hlr)
# Not using autotools
echo "0.0.1"
;;
osmo-mgw)
# openbsc
echo "1.0.1"
;;
osmo-msc)
# openbsc
echo "1.0.1"
;;
osmo-pcap)
# No rule to make target 'osmo-pcap-server.cfg', needed by 'distdir'
echo "0.0.3"
;;
osmo-pcu)
# Duplicates of 0.1.0, 0.2.0
echo "0.1"
echo "0.2"
;;
osmo-sgsn)
# openbsc
echo "0.9.0 0.9.1 0.9.2 0.9.3 0.9.4 0.9.5 0.9.6 0.9.8 0.9.9 0.9.10 0.9.11 0.9.12 0.9.13 0.9.14"
echo "0.9.15 0.9.16 0.10.0 0.10.1 0.11.0 0.12.0 0.13.0 0.14.0 0.15.0 1.0.1"
;;
osmo-sip-connector)
# make: *** No rule to make target 'osmoappdesc.py'
echo "0.0.1"
;;
osmo-trx)
# cp: cannot stat './/home/user/code/osmo-dev/src/osmo-ci/_temp/repos/osmo-trx/configure'
echo "0.2.0"
echo "0.3.0"
echo "1.3.0"
;;
esac
}
# Clone dependency repositories.
# $1: Osmocom repository
prepare_depends() {
case "$1" in
osmo-bts)
# Includes openbsc/gsm_data_shared.h
prepare_repo "openbsc"
;;
esac
}
# Run ./configure, with arguments if needed.
# $1: Osmocom repository
run_configure() {
case "$1" in
osmo-trx)
# Avoid pointing LIBTRXCON_DIR to an empty directory:
# https://gerrit.osmocom.org/c/osmo-trx/+/30792
if grep -q with_mstrx configure.ac; then
./configure --with-mstrx
fi
;;
*)
./configure
;;
esac
}
# Apply workarounds for bugs that break too many releases. This function runs between ./configure and make dist-bzip2.
# $1: Osmocom repository
fix_repo() {
case "$1" in
osmo-mgw)
# No rule to make target 'osmocom/mgcp_client/mgcp_common.h' (OS#4084)
make -C "$TEMP/repos/$1/include/osmocom/mgcp_client" mgcp_common.h || true
;;
esac
}
# Check if one specific tag should be ignored.
# $1: Osmocom repository
# $2: tag (e.g. "1.0.0")
ignore_tag() {
local repo="$1"
local tag="$2"
local tags="$(tags_to_ignore "$repo")"
for tag_i in $tags; do
if [ "$tag" = "$tag_i" ]; then
return 0
fi
done
return 1
}
# Delete existing temp dir (unless KEEP_TEMP is set). If all repos were checked out, this restores ~500 MB of space.
remove_temp_dir() {
if [ -n "$KEEP_TEMP" ]; then
echo "NOTE: not removing temp dir, because KEEP_TEMP is set: $TEMP"
elif [ -d "$TEMP" ]; then
rm -rf "$TEMP"
fi
}
get_existing_tarballs() {
if [ -n "$EXISTING_TARBALLS_FROM_JENKINS" ]; then
wget -O "$TEMP"/existing_tarballs \
https://jenkins.osmocom.org/jenkins/job/Osmocom-release-tarballs/ws/release-tarball-build-dist/_temp/existing_tarballs
elif ! $SSH_COMMAND releases@ftp.osmocom.org -- \
find web-files -name '*.tar.bz2' \
> "$TEMP"/existing_tarballs; then
echo "ERROR: getting existing tarballs from remote failed!"
exit 1
fi
}
# Clone an Osmocom repository to $TEMP/repos/$repo, clean it, checkout a tag.
# $1: Osmocom repository (may end in subdir, e.g. simtrace2/host)
# $2: tag (optional, default: master)
prepare_repo() {
local repo="$1"
local tag="${2:-master}"
if ! [ -d "$TEMP/repos/$repo" ]; then
git -C "$TEMP/repos" clone --recursive "$OSMO_GIT_URL/$repo"
fi
cd "$TEMP/repos/$repo"
git clean -qdxf
git reset --hard HEAD # in case the tracked files were modified (e.g. libsmpp34 1.10)
git checkout -q "$tag"
git submodule update --recursive
}
# Get the desired tarball name, replace / with - in $1.
# $1: Osmocom repository (may end in subdir, e.g. simtrace2/host)
# $2: tag
tarball_name() {
echo "$(echo "$repo" | tr / -)-$tag.tar.bz2"
}
# Checkout a given tag and build a release tarball.
# $1: Osmocom repository (may end in subdir, e.g. simtrace2/host)
# $2: tag
create_tarball() {
local repo="$1"
local tag="$2"
local tarball="$(tarball_name "$repo" "$tag")"
# Be verbose during the tarball build and preparation. Everything else is not verbose, so we can generate an
# easy to read overview of tarballs that are already built or are ignored.
set -x
prepare_repo "$repo" "$tag"
prepare_depends "$repo"
cd "$TEMP/repos/$repo"
autoreconf -fi
run_configure "$repo"
fix_repo "$repo"
make dist-bzip2
# Back to non-verbose mode
set +x
if ! [ -e "$tarball" ]; then
echo "NOTE: tarball has a different name (wrong version in configure.ac?), renaming."
mv -v *.tar.bz2 "$tarball"
fi
}
# Create a release tarball with "git archive" for non-autotools projects.
# $1: Osmocom repository
# $2: tag
create_tarball_git() {
local repo="$1"
local tag="$2"
local tarball="$(tarball_name "$repo" "$tag")"
set -x
cd "$TEMP/repos/$repo"
git archive \
-o "$tarball" \
"$tag"
set +x
}
# Move a generated release tarball to the output dir.
# $1: Osmocom repository (may end in subdir, e.g. simtrace2/host)
# $2: tag
move_tarball() {
local repo="$1"
local tag="$2"
local tarball="$(tarball_name "$repo" "$tag")"
local repo_dir="$(echo "$repo" | cut -d / -f 1)"
cd "$TEMP/repos/$repo"
mkdir -p "$OUTPUT/$repo_dir"
mv "$tarball" "$OUTPUT/$repo_dir/$tarball"
}
# Check if a git tag has a specific file
# $1: Osmocom repository
# $2: tag
# $3: file
tag_has_file() {
local repo="$1"
local tag="$2"
local file="$3"
git -C "$TEMP/repos/$repo" show "$tag:$file" >/dev/null 2>&1
}
# Create and move tarballs for Osmocom repositories.
# $1: Osmocom repository
# $2: tag
create_move_tarball() {
local repo="$1"
local tag="$2"
case "$repo" in
simtrace2)
if tag_has_file "$repo" "$tag" host/configure.ac; then
create_tarball "$repo/host" "$tag"
move_tarball "$repo/host" "$tag"
else
prepare_repo "$repo"
fi
create_tarball_git "$repo" "$tag"
move_tarball "$repo" "$tag"
;;
*)
create_tarball "$repo" "$tag"
move_tarball "$repo" "$tag"
;;
esac
}
upload() {
if ! [ -d _release_tarballs ]; then
echo "upload: no tarballs generated, nothing to do."
return
fi
cd _release_tarballs
rsync -avz -e "$SSH_COMMAND" . releases@ftp.osmocom.org:web-files/
}
remove_temp_dir
mkdir -p "$TEMP/repos"
echo "Temp dir: $TEMP"
get_existing_tarballs
for repo in $OSMO_RELEASE_REPOS; do
echo "$repo"
tags="$(osmo_git_last_commits_tags "$repo" "all" | cut -d / -f 3)"
# Skip untagged repos
if [ -z "$tags" ]; then
echo " (repository has no release tags)"
continue
fi
# Build missing tarballs for each tag
for tag in $tags; do
tarball="$repo-$tag.tar.bz2"
if ignore_tag "$repo" "$tag"; then
echo " $tarball (ignored)"
continue
elif [ -e "$OUTPUT/$repo/$tarball" ]; then
echo " $tarball (exists locally)"
continue
elif grep -q "^web-files/$repo/$tarball$" "$TEMP"/existing_tarballs; then
echo " $tarball (exists on server)"
continue
fi
echo " $tarball (creating)"
create_move_tarball "$repo" "$tag"
done
done
cd /build
remove_temp_dir
upload
echo "done!"
|