Skip to content

Commit a5a47b7

Browse files
uparrowsuparrows
committed
Upgrade to 0.7.8
1 parent dcda9fb commit a5a47b7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+1484
-691
lines changed

‎Dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ COPY /tools/build/docker/s6/services.d/ /etc/services.d/
6565
COPY --chown=root:root /jquery.dataTables.min.js /root/lanraragi/public/js/vendor/jquery.dataTables.min.js
6666
COPY --chown=root:root /jquery-migrate.min.map /root/lanraragi/public/js/vendor/jquery-migrate.min.map
6767
COPY --chown=root:root /awesomplete.min.js.map /root/lanraragi/public/js/vendor/awesomplete.min.js.map
68-
COPY --chown=root:root /jquery.contextMenu.min.js.map /root/lanraragi/public/js/vendor/jquery.contextMenu.min.js.map
6968
COPY --chown=root:root /jquery.qtip.min.map /root/lanraragi/public/js/vendor/jquery.qtip.min.map
7069
COPY --chown=root:root /jquery.contextMenu.min.css.map /root/lanraragi/public/css/vendor/jquery.contextMenu.min.css.map
7170
COPY --chown=root:root /awesomplete.css.map /root/lanraragi/public/css/vendor/awesomplete.css.map

‎lib/LANraragi.pm‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,12 @@ sub startup {
126126
# Enable Minion capabilities in the app
127127
shutdown_from_pid("./script/minion.pid");
128128

129-
#Delete old DB if it still exists -- Might be needed in case of DB lock ?
129+
# Delete old SQLite DB if it still exists
130130
unlink("./.minion.db");
131131

132-
$self->plugin( 'Minion' => { SQLite => 'sqlite:./.minion.db' } );
132+
my $miniondb = $self->LRR_CONF->get_redisad . "/" . $self->LRR_CONF->get_miniondb;
133+
say "Minion will use the Redis database at $miniondb";
134+
$self->plugin( 'Minion' => { Redis => "redis://$miniondb" } );
133135
$self->LRR_LOGGER->info("Successfully connected to Minion database.");
134136
$self->minion->missing_after(5); # Clean up older workers after 5 seconds of unavailability
135137

@@ -138,8 +140,9 @@ sub startup {
138140

139141
# Warm search cache
140142
# /!\ Enqueuing tasks must be done either before starting the worker, or once the IOLoop is started!
141-
# Anything else will cause weird database lockups with the SQLite Minion backend.
143+
# Anything else can cause weird database lockups.
142144
$self->minion->enqueue('warm_cache');
145+
$self->minion->enqueue('build_stat_hashes');
143146

144147
# Start a Minion worker in a subprocess
145148
start_minion($self);

‎lib/LANraragi/Controller/Api/Archive.pm‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use Mojo::JSON qw(decode_json encode_json from_json);
88
use Scalar::Util qw(looks_like_number);
99

1010
use LANraragi::Utils::Generic qw(render_api_response);
11+
use LANraragi::Utils::Database;
1112

1213
use LANraragi::Model::Archive;
1314
use LANraragi::Model::Category;
@@ -147,6 +148,22 @@ sub clear_new {
147148
);
148149
}
149150

151+
sub delete_archive {
152+
my $self = shift;
153+
my $id = check_id_parameter( $self, "delete_archive" ) || return;
154+
155+
my $delStatus = LANraragi::Utils::Database::delete_archive($id);
156+
157+
$self->render(
158+
json => {
159+
operation => "delete_archive",
160+
id => $id,
161+
filename => $delStatus,
162+
success => $delStatus eq "0" ? 0 : 1
163+
}
164+
);
165+
}
166+
150167
sub update_metadata {
151168
my $self = shift;
152169
my $id = check_id_parameter( $self, "update_metadata" ) || return;

‎lib/LANraragi/Controller/Api/Other.pm‎

Lines changed: 44 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
package LANraragi::Controller::Api::Other;
22
use Mojo::Base 'Mojolicious::Controller';
33

4-
use Mojo::JSON qw(encode_json);
4+
use Mojo::JSON qw(encode_json decode_json);
55
use Redis;
66

77
use LANraragi::Model::Stats;
88
use LANraragi::Utils::TempFolder qw(get_tempsize clean_temp_full);
99
use LANraragi::Utils::Generic qw(render_api_response);
10-
use LANraragi::Utils::Plugins qw(get_plugin get_plugins get_plugin_parameters);
10+
use LANraragi::Utils::Plugins qw(get_plugin get_plugins get_plugin_parameters use_plugin);
1111

1212
sub serve_serverinfo {
1313
my $self = shift;
@@ -128,51 +128,65 @@ sub download_url {
128128
}
129129

130130
# Uses a plugin, with the standard global arguments and a provided oneshot argument.
131-
sub use_plugin {
131+
sub use_plugin_sync {
132132

133133
my ($self) = shift;
134134
my $id = $self->req->param('id') || 0;
135135
my $plugname = $self->req->param('plugin');
136136
my $input = $self->req->param('arg');
137137

138-
my $plugin = get_plugin($plugname);
139-
my %plugin_result;
140-
my %pluginfo;
138+
my ( $pluginfo, $plugin_result ) = use_plugin( $plugname, $id, $input );
141139

142-
if ( !$plugin ) {
143-
$plugin_result{error} = "Plugin not found on system.";
144-
} else {
145-
%pluginfo = $plugin->plugin_info();
140+
#Returns the fetched tags in a JSON response.
141+
$self->render(
142+
json => {
143+
operation => "use_plugin",
144+
type => $pluginfo->{type},
145+
success => ( exists $plugin_result->{error} ? 0 : 1 ),
146+
error => $plugin_result->{error},
147+
data => $plugin_result
148+
}
149+
);
150+
return;
151+
}
146152

147-
#Get the plugin settings in Redis
148-
my @settings = get_plugin_parameters($plugname);
153+
# Queues a plugin execution into Minion.
154+
sub use_plugin_async {
149155

150-
#Execute the plugin, appending the custom args at the end
151-
if ( $pluginfo{type} eq "script" ) {
152-
eval { %plugin_result = LANraragi::Model::Plugins::exec_script_plugin( $plugin, $input, @settings ); };
153-
}
156+
my ($self) = shift;
157+
my $id = $self->req->param('id') || 0;
158+
my $priority = $self->req->param('priority') || 0;
159+
my $plugname = $self->req->param('plugin');
160+
my $input = $self->req->param('arg');
154161

155-
if ( $pluginfo{type} eq "metadata" ) {
156-
eval { %plugin_result = LANraragi::Model::Plugins::exec_metadata_plugin( $plugin, $id, $input, @settings ); };
157-
}
162+
my $jobid = $self->minion->enqueue( run_plugin => [ $plugname, $id, $input ] => { priority => $priority } );
158163

159-
if ($@) {
160-
$plugin_result{error} = $@;
164+
$self->render(
165+
json => {
166+
operation => "queue_plugin_exec",
167+
success => 1,
168+
job => $jobid
161169
}
162-
}
170+
);
171+
}
172+
173+
# Queues a job into Minion.
174+
sub queue_minion_job {
175+
176+
my ($self) = shift;
177+
my $jobname = $self->stash('jobname');
178+
my @jobargs = decode_json( $self->req->param('args') );
179+
my $priority = $self->req->param('priority') || 0;
180+
181+
my $jobid = $self->minion->enqueue( $jobname => @jobargs => { priority => $priority } );
163182

164-
#Returns the fetched tags in a JSON response.
165183
$self->render(
166184
json => {
167-
operation => "use_plugin",
168-
type => $pluginfo{type},
169-
success => ( exists $plugin_result{error} ? 0 : 1 ),
170-
error => $plugin_result{error},
171-
data => \%plugin_result
185+
operation => "queue_minion_job",
186+
success => 1,
187+
job => $jobid
172188
}
173189
);
174-
return;
175-
176190
}
177191

178192
1;

‎lib/LANraragi/Controller/Batch.pm‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ use Mojo::Base 'Mojolicious::Controller';
33

44
use Redis;
55
use Encode;
6-
use Mojo::IOLoop::Subprocess;
76
use Mojo::Util qw(xml_escape);
87
use Mojo::JSON qw(decode_json encode_json from_json);
98

‎lib/LANraragi/Controller/Edit.pm‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,6 @@ use LANraragi::Utils::Generic qw(generate_themes_selector generate_themes_header
1111
use LANraragi::Utils::Database qw(redis_decode);
1212
use LANraragi::Utils::Plugins qw(get_plugins);
1313

14-
sub delete_archive {
15-
my $self = shift;
16-
my $id = $self->req->param('id');
17-
18-
my $delStatus = LANraragi::Utils::Database::delete_archive($id);
19-
20-
$self->render(
21-
json => {
22-
id => $id,
23-
operation => "delete",
24-
success => $delStatus
25-
}
26-
);
27-
}
28-
2914
sub index {
3015
my $self = shift;
3116

‎lib/LANraragi/Controller/Reader.pm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ sub index {
7575
$self->render(
7676
template => "reader",
7777
arcname => $arcname,
78+
tags => $tags,
7879
id => $id,
7980
imgpaths => $imgpaths,
8081
filename => $filename,

‎lib/LANraragi/Model/Config.pm‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,16 @@ sub get_redisad { return $config->{redis_address} }
2222
#Database that'll be used by LANraragi. Redis databases are numbered, default is 0.
2323
sub get_redisdb { return $config->{redis_database} }
2424

25+
#Database that'll be used by Minion. Redis databases are numbered, default is 0.
26+
sub get_miniondb { return $config->{redis_database_minion} }
27+
2528
#Default CSS file to load.
2629
sub get_style { return $config->{default_theme} }
2730

2831
# Create a Minion object connected to the Minion database.
2932
sub get_minion {
30-
return Minion->new( SQLite => 'sqlite:' . $home . '/.minion.db' );
33+
my $miniondb = get_redisad . "/" . get_miniondb;
34+
return Minion->new( Redis => "redis://$miniondb" );
3135
}
3236

3337
#get_redis

‎lib/LANraragi/Model/Stats.pm‎

Lines changed: 68 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ use utf8;
77
use Redis;
88
use File::Find;
99

10-
use LANraragi::Utils::Generic qw(remove_spaces remove_newlines is_archive);
11-
use LANraragi::Utils::Database qw(redis_decode);
10+
use LANraragi::Utils::Generic qw(remove_spaces remove_newlines is_archive trim_url);
11+
use LANraragi::Utils::Database qw(redis_decode redis_encode);
1212
use LANraragi::Utils::Logging qw(get_logger);
1313

1414
sub get_archive_count {
@@ -43,44 +43,90 @@ sub get_page_stat {
4343
return $stat;
4444
}
4545

46-
sub build_tag_json {
47-
48-
my $t;
49-
my @tags;
50-
my %tagcloud;
46+
# This operation builds two hashes: LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag,
47+
# and LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON.
48+
sub build_stat_hashes {
5149

52-
#Login to Redis and get all hashes
53-
my $redis = LANraragi::Model::Config->get_redis;
54-
my $logger = get_logger( "Tag Stats", "lanraragi" );
50+
# This method does only one atomic write transaction, using Redis' watch/multi mode.
51+
# But we can't use the connection to get other data while it's in transaction mode! So we instantiate a second connection to get the data we need.
52+
my $redis = LANraragi::Model::Config->get_redis;
53+
my $redistx = LANraragi::Model::Config->get_redis;
54+
my $logger = get_logger( "Tag Stats", "lanraragi" );
5555

56-
#40-character long keys only => Archive IDs
56+
# 40-character long keys only => Archive IDs
5757
my @keys = $redis->keys('????????????????????????????????????????');
5858

59-
#Iterate on hashes to get their tags
59+
# Cancel the transaction if the hashes have been modified by another job in the meantime.
60+
# This also allows for the previous stats/map to still be readable until we're done.
61+
$redistx->watch( "LRR_STATS", "LRR_URLMAP" );
62+
$redistx->multi;
63+
$redistx->del("LRR_STATS");
64+
$redistx->del("LRR_URLMAP");
65+
66+
# Iterate on hashes to get their tags
67+
$logger->debug("Building stat indexes...");
6068
foreach my $id (@keys) {
6169
if ( $redis->hexists( $id, "tags" ) ) {
6270

63-
$t = $redis->hget( $id, "tags" );
64-
$t = redis_decode($t);
71+
my $rawtags = $redis->hget( $id, "tags" );
6572

6673
#Split tags by comma
67-
@tags = split( /,\s?/, $t );
74+
my @tags = split( /,\s?/, redis_decode($rawtags) );
6875

6976
foreach my $t (@tags) {
70-
7177
remove_spaces($t);
7278
remove_newlines($t);
7379

74-
#Increment value of tag or create it
75-
if ( exists( $tagcloud{$t} ) ) { $tagcloud{$t}++; }
76-
else { $tagcloud{$t} = 1; }
77-
}
80+
# If the tag is a source: tag, add it to the URL index
81+
if ( $t =~ /source:(.*)/i ) {
82+
my $url = $1;
83+
$logger->debug("Adding $url as an URL for $id");
84+
trim_url($url);
85+
$logger->debug("Trimmed: $url");
86+
$redistx->hset( "LRR_URLMAP", $url, $id ); # No need to encode the value, as URLs are already encoded by design
87+
}
7888

89+
# Increment tag in stats, all lowercased here to avoid redundancy/dupes
90+
$redistx->zincrby( "LRR_STATS", 1, redis_encode( lc($t) ) );
91+
}
7992
}
8093
}
94+
95+
$redistx->exec;
96+
$logger->debug("Done!");
97+
$redis->quit;
98+
$redistx->quit;
99+
}
100+
101+
sub is_url_recorded {
102+
103+
my $url = $_[0];
104+
my $logger = get_logger( "Tag Stats", "lanraragi" );
105+
my $redis = LANraragi::Model::Config->get_redis;
106+
my $id = 0;
107+
$logger->debug("Checking if url $url is in the url map.");
108+
109+
# Trim last slash from url if it's present
110+
trim_url($url);
111+
112+
if ( $redis->hexists( "LRR_URLMAP", $url ) ) {
113+
$id = $redis->hget( "LRR_URLMAP", $url );
114+
$logger->debug("Found! id $id.");
115+
}
116+
$redis->quit;
117+
return $id;
118+
}
119+
120+
sub build_tag_json {
121+
122+
my $logger = get_logger( "Tag Stats", "lanraragi" );
123+
124+
#Login to Redis and grab the stats sorted set
125+
my $redis = LANraragi::Model::Config->get_redis;
126+
my %tagcloud = $redis->zrange( "LRR_STATS", 0, -1, "WITHSCORES" );
81127
$redis->quit();
82128

83-
#Go through the tagCloud hash and build a JSON
129+
# Go through the data from stats and build a JSON
84130
my $tagsjson = "[";
85131

86132
for ( keys %tagcloud ) {
@@ -89,7 +135,7 @@ sub build_tag_json {
89135
# Split namespace
90136
# detect the : symbol and only use what's after it
91137
my $ns = "";
92-
my $t = $_;
138+
my $t = redis_decode($_);
93139
if ( $t =~ /(.*):(.*)/ ) { $ns = $1; $t = $2; }
94140

95141
if ( $_ ne "" ) {

‎lib/LANraragi/Model/Upload.pm‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use File::Copy qw(move);
1313
use LANraragi::Utils::Database qw(invalidate_cache compute_id);
1414
use LANraragi::Utils::Logging qw(get_logger);
1515
use LANraragi::Utils::Database qw(redis_encode);
16-
use LANraragi::Utils::Generic qw(is_archive);
16+
use LANraragi::Utils::Generic qw(is_archive remove_spaces remove_newlines trim_url);
1717

1818
use LANraragi::Model::Config;
1919
use LANraragi::Model::Plugins;
@@ -76,6 +76,24 @@ sub handle_incoming_file {
7676
# If additional tags were given to the sub, add them now.
7777
if ($tags) {
7878
$redis->hset( $id, "tags", redis_encode($tags) );
79+
80+
# Check for a source: tag, and if it exists amend the urlmap by hand.
81+
# This is faster than queueing a full recalculation job.
82+
my @tags = split( /,\s?/, $tags );
83+
84+
foreach my $t (@tags) {
85+
remove_spaces($t);
86+
remove_newlines($t);
87+
88+
# If the tag is a source: tag, add it to the URL index
89+
if ( $t =~ /source:(.*)/i ) {
90+
my $url = $1;
91+
$logger->debug("Adding $url as an URL for $id");
92+
trim_url($url);
93+
$logger->debug("Trimmed: $url");
94+
$redis->hset( "LRR_URLMAP", $url, $id ); # No need to encode the value, as URLs are already encoded by design
95+
}
96+
}
7997
}
8098

8199
$redis->quit();

0 commit comments

Comments
 (0)