Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

"Upcoming meetups" only show one event per group #532 #566

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

"Upcoming meetups" only show one event per group #532 #566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dd9f877
#532 Change to return all meetups in the next 30 days
jameswilki Apr 11, 2020
06232b9
Added try-catch to handle Invalid responses to aid debugging
jameswilki Apr 11, 2020
0ea9cdf
Added try-catch to handle Invalid responses to aid debugging
jameswilki Apr 11, 2020
26a60fb
Added null check on results before returning tweets to aid debugging
jameswilki Apr 11, 2020
7da0f3d
#532 Increased the time-span to 60 minutes to reduce request count
jameswilki Apr 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 63 additions & 54 deletions OurUmbraco/Community/Meetup/MeetupService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,60 +25,67 @@ public class MeetupService
{
public void UpdateMeetupStats()
{
var configPath = HostingEnvironment.MapPath("~/config/MeetupUmbracoGroups.txt");
// Get the alias (urlname) of each group from the config file
var aliases = File.ReadAllLines(configPath).Where(x => x.Trim() != "").Distinct().ToArray();

var counterPath = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCounter.txt");
var counter = 0;
if (File.Exists(counterPath))
try
{
var savedCounter = File.ReadAllLines(counterPath).First();
int.TryParse(savedCounter, out counter);
}
var configPath = HostingEnvironment.MapPath("~/config/MeetupUmbracoGroups.txt");
// Get the alias (urlname) of each group from the config file
var aliases = File.ReadAllLines(configPath).Where(x => x.Trim() != "").Distinct().ToArray();

var newCounter = aliases.Length <= counter ? 0 : counter + 1;
File.WriteAllText(counterPath, newCounter.ToString(), Encoding.UTF8);
var counterPath = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCounter.txt");
var counter = 0;
if (File.Exists(counterPath))
{
var savedCounter = File.ReadAllLines(counterPath).First();
int.TryParse(savedCounter, out counter);
}

var client = new MeetupOAuth2Client();
var response = client.DoHttpGetRequest(string.Format("https://api.meetup.com/{0}/events?page=1000&status=past", aliases[counter]));
var events = MeetupGetEventsResponse.ParseResponse(response).Body;
var newCounter = aliases.Length <= counter ? 0 : counter + 1;
File.WriteAllText(counterPath, newCounter.ToString(), Encoding.UTF8);

var meetupCache = new List<MeetupCacheItem>();
var meetupCacheFile = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCache.json");
if (File.Exists(meetupCacheFile))
{
var json = File.ReadAllText(meetupCacheFile);
using (var stringReader = new StringReader(json))
using (var jsonTextReader = new JsonTextReader(stringReader))
var client = new MeetupOAuth2Client();
var response = client.DoHttpGetRequest(string.Format("https://api.meetup.com/{0}/events?page=1000&status=past", aliases[counter]));
var events = MeetupGetEventsResponse.ParseResponse(response).Body;

var meetupCache = new List<MeetupCacheItem>();
var meetupCacheFile = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCache.json");
if (File.Exists(meetupCacheFile))
{
var jsonSerializer = new JsonSerializer();
meetupCache = jsonSerializer.Deserialize<List<MeetupCacheItem>>(jsonTextReader);
var json = File.ReadAllText(meetupCacheFile);
using (var stringReader = new StringReader(json))
using (var jsonTextReader = new JsonTextReader(stringReader))
{
var jsonSerializer = new JsonSerializer();
meetupCache = jsonSerializer.Deserialize<List<MeetupCacheItem>>(jsonTextReader);
}
}
}

foreach (var meetupEvent in events)
{
if (meetupCache.Any(x => x.Id == meetupEvent.Id))
continue;

var meetupCacheItem = new MeetupCacheItem
foreach (var meetupEvent in events)
{
Time = meetupEvent.Time,
Created = meetupEvent.Created,
Description = meetupEvent.Description,
HasVenue = meetupEvent.HasVenue,
Id = meetupEvent.Id,
Link = meetupEvent.Link,
Name = meetupEvent.Name,
Updated = meetupEvent.Updated,
Visibility = meetupEvent.Visibility
};
meetupCache.Add(meetupCacheItem);
}
if (meetupCache.Any(x => x.Id == meetupEvent.Id))
continue;

var rawJson = JsonConvert.SerializeObject(meetupCache, Formatting.Indented);
File.WriteAllText(meetupCacheFile, rawJson, Encoding.UTF8);
var meetupCacheItem = new MeetupCacheItem
{
Time = meetupEvent.Time,
Created = meetupEvent.Created,
Description = meetupEvent.Description,
HasVenue = meetupEvent.HasVenue,
Id = meetupEvent.Id,
Link = meetupEvent.Link,
Name = meetupEvent.Name,
Updated = meetupEvent.Updated,
Visibility = meetupEvent.Visibility
};
meetupCache.Add(meetupCacheItem);
}

var rawJson = JsonConvert.SerializeObject(meetupCache, Formatting.Indented);
File.WriteAllText(meetupCacheFile, rawJson, Encoding.UTF8);
}
catch (Exception ex)
{
LogHelper.Error<MeetupsController>("Could not get events from meetup.com", ex);
}
}


Expand All @@ -95,6 +102,7 @@ public MeetupEventsModel GetUpcomingMeetups()
if (File.Exists(configPath) == false)
{
LogHelper.Debug<MeetupsController>("Config file was not found: " + configPath);

return meetups;
}

Expand All @@ -106,7 +114,6 @@ public MeetupEventsModel GetUpcomingMeetups()
{
// Initialize a new service instance (we don't specify an API key since we're accessing public data)
var service = new Skybrud.Social.Meetup.MeetupService();

var items = new List<MeetupItem>();

foreach (var alias in aliases)
Expand All @@ -119,17 +126,19 @@ public MeetupEventsModel GetUpcomingMeetups()
if (meetupGroup.JObject.HasValue("next_event") == false)
continue;

var nextEventId = meetupGroup.JObject.GetString("next_event.id");

// Make the call to the Meetup.com API to get upcoming events
var events = service.Events.GetEvents(alias);

// Get the next event(s)
var nextEvent = events.Body.FirstOrDefault(x => x.Id == nextEventId);
// Get the events in the next 30 days
var nextEvents = events.Body.Where(x => x.Time < DateTime.Now.AddDays(30) );

// Append the first event of the group
if (nextEvent != null)
items.Add(new MeetupItem(meetupGroup, nextEvent));
if (nextEvents.Any())
{
foreach (var item in nextEvents)
{
items.Add(new MeetupItem(meetupGroup, item));
}
}
}
catch (Exception ex)
{
Expand All @@ -139,7 +148,7 @@ public MeetupEventsModel GetUpcomingMeetups()

return items.OrderBy(x => x.Event.Time).ToArray();

}, TimeSpan.FromMinutes(30));
}, TimeSpan.FromMinutes(60));
}
catch (Exception ex)
{
Expand Down
2 changes: 1 addition & 1 deletion OurUmbraco/Community/Twitter/TwitterService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public TweetsModel GetTweets(int numberOfResults, bool adminOverview)
};

var results = service.Search(options);
return results.Statuses;
return results?.Statuses;

}, TimeSpan.FromMinutes(2));

Expand Down
148 changes: 78 additions & 70 deletions OurUmbraco/Community/Videos/CommunityVideosService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,103 +96,111 @@ public void UpdateYouTubePlaylistVideos()

public void UpdateYouTubePlaylistVideo(string playlistId)
{

// Make sure we have a TEMP directory
string dir = IOHelper.MapPath("~/App_Data/TEMP/YouTube/");
Directory.CreateDirectory(dir);

// Make an initial request to get information about the playlist
var response1 = Api.YouTube.Playlists.GetPlaylists(new YouTubeGetPlaylistListOptions
try
{
Ids = new[] { playlistId }
});
// Make sure we have a TEMP directory
string dir = IOHelper.MapPath("~/App_Data/TEMP/YouTube/");
Directory.CreateDirectory(dir);

// Get a reference to the playlist (using "Single" as there should be exactly one playlist)
var playlist = response1.Body.Items.Single();

// Save the playlist to the disk
JsonUtils.SaveJsonObject(dir + "Playlist_" + playlist.Id + ".json", playlist);
// Make an initial request to get information about the playlist
var response1 = Api.YouTube.Playlists.GetPlaylists(new YouTubeGetPlaylistListOptions
{
Ids = new[] {playlistId}
});

// List of all video IDs
List<string> ids = new List<string>();
// Get a reference to the playlist (using "Single" as there should be exactly one playlist)
var playlist = response1.Body.Items.Single();

// Initialize the options for getting the playlist items
var playlistItemsOptions = new YouTubeGetPlaylistItemListOptions(playlistId)
{
// Save the playlist to the disk
JsonUtils.SaveJsonObject(dir + "Playlist_" + playlist.Id + ".json", playlist);

// Maximum allowed value is 50
MaxResults = 50
// List of all video IDs
List<string> ids = new List<string>();

};
// Initialize the options for getting the playlist items
var playlistItemsOptions = new YouTubeGetPlaylistItemListOptions(playlistId)
{

int page = 0;
while (page < 10)
{
// Maximum allowed value is 50
MaxResults = 50

// Get the playlist items
var response2 = Api.YouTube.PlaylistItems.GetPlaylistItems(playlistItemsOptions);
};

// Append each video ID to the list
foreach (var item in response2.Body.Items)
int page = 0;
while (page < 10)
{
ids.Add(item.VideoId);
}

// Break the loop if there are no additional pages
if (String.IsNullOrWhiteSpace(response2.Body.NextPageToken))
{
break;
}
// Get the playlist items
var response2 = Api.YouTube.PlaylistItems.GetPlaylistItems(playlistItemsOptions);

// Update the options with the page token
playlistItemsOptions.PageToken = response2.Body.NextPageToken;
// Append each video ID to the list
foreach (var item in response2.Body.Items)
{
ids.Add(item.VideoId);
}

page++;
// Break the loop if there are no additional pages
if (String.IsNullOrWhiteSpace(response2.Body.NextPageToken))
{
break;
}

}
// Update the options with the page token
playlistItemsOptions.PageToken = response2.Body.NextPageToken;

// Iterate through groups of IDs (maximum 50 items per group)
foreach (var group in ids.Where(x => !ExistsOnDiskAndIsUpToDate(x)).InGroupsOf(50))
{
page++;

}

// Initialize the video options
var videosOptions = new YouTubeGetVideoListOptions
// Iterate through groups of IDs (maximum 50 items per group)
foreach (var group in ids.Where(x => !ExistsOnDiskAndIsUpToDate(x)).InGroupsOf(50))
{
Ids = group.ToArray(),
Part = YouTubeVideoParts.Snippet + YouTubeVideoParts.ContentDetails + YouTubeVideoParts.Statistics
};

// Make a request to the APi to get video information
var res3 = Api.YouTube.Videos.GetVideos(videosOptions);
// Initialize the video options
var videosOptions = new YouTubeGetVideoListOptions
{
Ids = group.ToArray(),
Part = YouTubeVideoParts.Snippet + YouTubeVideoParts.ContentDetails +
YouTubeVideoParts.Statistics
};

// Iterate through the videos
foreach (var video in res3.Body.Items)
{
// Make a request to the APi to get video information
var res3 = Api.YouTube.Videos.GetVideos(videosOptions);

// Save the video to the disk
string path = dir + "Video_" + video.Id + ".json";
JsonUtils.SaveJsonObject(path, video);
// Iterate through the videos
foreach (var video in res3.Body.Items)
{

// Download the thumnails for each video
var thumbnailUrl = GetThumbnail(video).Url;
// Save the video to the disk
string path = dir + "Video_" + video.Id + ".json";
JsonUtils.SaveJsonObject(path, video);

const string mediaRoot = "~/media/YouTube";
var thumbnailFile = IOHelper.MapPath($"{mediaRoot}/{video.Id}.jpg");
// Download the thumnails for each video
var thumbnailUrl = GetThumbnail(video).Url;

var mediaPath = IOHelper.MapPath(mediaRoot);
if (Directory.Exists(mediaPath) == false)
Directory.CreateDirectory(mediaPath);
const string mediaRoot = "~/media/YouTube";
var thumbnailFile = IOHelper.MapPath($"{mediaRoot}/{video.Id}.jpg");

if (File.Exists(thumbnailFile))
continue;
var mediaPath = IOHelper.MapPath(mediaRoot);
if (Directory.Exists(mediaPath) == false)
Directory.CreateDirectory(mediaPath);

using (var client = new WebClient())
client.DownloadFile(thumbnailUrl, thumbnailFile);
if (File.Exists(thumbnailFile))
continue;

using (var client = new WebClient())
client.DownloadFile(thumbnailUrl, thumbnailFile);
}
}
}

// Load the videos from the individual files, and save them to a common file
JsonUtils.SaveJsonArray(dir + "Playlist_" + playlistId + "_Videos.json", ids.Select(LoadYouTubeVideo).WhereNotNull());
// Load the videos from the individual files, and save them to a common file
JsonUtils.SaveJsonArray(dir + "Playlist_" + playlistId + "_Videos.json",
ids.Select(LoadYouTubeVideo).WhereNotNull());
}
catch (Exception ex)
{
LogHelper.Error<CommunityVideosService>("Could not get data from YouTube", ex);
}
}

private static YouTubeVideoThumbnail GetThumbnail(YouTubeVideo video)
Expand Down
Loading